<commit_before>extern crate serialize;

use std::collections::HashMap;
use std::io::process::Command;
use std::io::{File, TempDir};
use std::io::fs;

fn main() {
    let (args, passthrough) = parse_arguments();

    // Find all the native shared libraries that exist in the target directory.
    let native_shared_libs = find_native_libs(&args);

    // getting the path from the ANDROID_HOME env
    let sdk_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "ANDROID_HOME")
        .map(|(_, v)| Path::new(v)).expect("Please set the ANDROID_HOME environment variable");

    // hardcoding ndk path
    let ndk_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "NDK_HOME")
        .map(|(_, v)| Path::new(v)).expect("Please set the NDK_HOME environment variable");

    // hardcoding ndk path
    let standalone_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "NDK_STANDALONE")
        .map(|(_, v)| Path::new(v)).unwrap_or(Path::new("/opt/ndk_standalone"));

    // creating the build directory that will contain all the necessary files to create teh apk
    let directory = build_directory(&sdk_path, args.output.filestem_str().unwrap(), &native_shared_libs);

    // Copy the additional native libs into the libs directory.
    for (name, path) in native_shared_libs.iter() {
        fs::copy(path, &directory.path().join("libs").join("armeabi").join(name)).unwrap();
    }

    // compiling android_native_app_glue.c
    if Command::new(standalone_path.join("bin").join("arm-linux-androideabi-gcc"))
        .arg(ndk_path.join("sources").join("android").join("native_app_glue").join("android_native_app_glue.c"))
        .arg("-c")
        .arg("-o").arg(directory.path().join("android_native_app_glue.o"))
        .stdout(std::io::process::InheritFd(1)).stderr(std::io::process::InheritFd(2))
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing gcc");
        std::os::set_exit_status(1);
        return;
    }

    // calling gcc to link to a shared object
    if Command::new(standalone_path.join("bin").join("arm-linux-androideabi-gcc"))
        .args(passthrough.as_slice())
        .arg(directory.path().join("android_native_app_glue.o"))
        .arg("-o").arg(directory.path().join("libs").join("armeabi").join("libmain.so"))
        .arg("-shared")
        .arg("-Wl,-E")
        .stdout(std::io::process::InheritFd(1))
        .stderr(std::io::process::InheritFd(2))//.cwd(directory.path())
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing gcc");
        std::os::set_exit_status(1);
        return;
    }

    // calling objdump to make sure that our object has `ANativeActivity_onCreate`
    // TODO: not working
    /*{
        let mut process =
            Command::new(standalone_path.join("bin").join("arm-linux-androideabi-objdump"))
            .arg("-x").arg(directory.path().join("libs").join("armeabi").join("libmain.so"))
            .stderr(std::io::process::InheritFd(2))
            .spawn().unwrap();

        // TODO: use UFCS instead
        fn by_ref<'a, T: Reader>(r: &'a mut T) -> std::io::RefReader<'a, T> { r.by_ref() };

        let stdout = process.stdout.as_mut().unwrap();
        let mut stdout = std::io::BufferedReader::new(by_ref(stdout));

        if stdout.lines().filter_map(|l| l.ok())
            .find(|line| line.as_slice().contains("ANativeActivity_onCreate")).is_none()
        {
            println!("Error: the output file doesn't contain ANativeActivity_onCreate");
            std::os::set_exit_status(1);
            return;
        }
    }*/

    // executing ant
    if Command::new("ant").arg("debug").stdout(std::io::process::InheritFd(1))
        .stderr(std::io::process::InheritFd(2)).cwd(directory.path())
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing ant debug");
        std::os::set_exit_status(1);
        return;
    }

    // copying apk file to the requested output
    fs::copy(&directory.path().join("bin").join("rust-android-debug.apk"),
        &args.output).unwrap();
}

struct Args {
    output: Path,
}

fn parse_arguments() -> (Args, Vec<String>) {
    let mut result_output = None;
    let mut result_passthrough = Vec::new();

    let args = std::os::args();
    let mut args = args.into_iter().skip(1);

    loop {
        let arg = match args.next() {
            None => return (
                Args {
                    output: result_output.expect("Could not find -o argument")
                },
                result_passthrough
            ),
            Some(arg) => arg
        };

        match arg.as_slice() {
            "-o" => {
                result_output = Some(Path::new(args.next().expect("-o must be followed by the output name")));
            },
            _ => result_passthrough.push(arg)
        };
    }
}

fn find_native_libs(args: &Args) -> HashMap<String, Path> {
    let base_path = args.output.dir_path().join("native");
    let mut native_shared_libs: HashMap<String, Path> = HashMap::new();

    fs::walk_dir(&base_path).and_then(|mut dirs| {
        for dir in dirs {
            fs::readdir(&dir).and_then(|paths| {
                for path in paths.iter() {
                    match (path.filename_str(), path.extension_str()) {
                        (Some(filename), Some(ext)) => {
                            if filename.starts_with("lib") && ext == "so" {
                                native_shared_libs.insert(filename.to_string(), path.clone());
                            }
                        }
                        _ => {}
                    }
                }
                Ok(())
            }).ok();
        }
        Ok(())
    }).ok();

    native_shared_libs
}

fn build_directory(sdk_dir: &Path, crate_name: &str, libs: &HashMap<String, Path>) -> TempDir {
    use std::io::fs;

    let build_directory = TempDir::new("android-rs-glue-rust-to-apk")
        .ok().expect("Could not create temporary build directory");

    let activity_name = if libs.len() > 0 {
        let src_path = build_directory.path().join("src/rust/glutin");
        fs::mkdir_recursive(&src_path, std::io::USER_RWX).unwrap();

        File::create(&src_path.join("MainActivity.java")).unwrap()
            .write_str(java_src(libs).as_slice())
            .unwrap();

        "rust.glutin.MainActivity"
    } else {
        "android.app.NativeActivity"
    };

    File::create(&build_directory.path().join("AndroidManifest.xml")).unwrap()
        .write_str(build_manifest(crate_name, activity_name).as_slice())
        .unwrap();

    File::create(&build_directory.path().join("build.xml")).unwrap()
        .write_str(build_build_xml().as_slice())
        .unwrap();

    File::create(&build_directory.path().join("local.properties")).unwrap()
        .write_str(build_local_properties(sdk_dir).as_slice())
        .unwrap();

    File::create(&build_directory.path().join("project.properties")).unwrap()
        .write_str(build_project_properties().as_slice())
        .unwrap();

    {
        let libs_path = build_directory.path().join("libs").join("armeabi");
        fs::mkdir_recursive(&libs_path, std::io::USER_RWX).unwrap();
    }

    build_directory
}

fn java_src(libs: &HashMap<String, Path>) -> String {
    let mut libs_string = "".to_string();

    for (name, _) in libs.iter() {
        // Strip off the 'lib' prefix and ".so" suffix. This is safe since libs only get added
        // to the hash map if they start with lib.
        let line = format!("        System.loadLibrary(\"{}\");\n", name.slice(3, name.len()-3));
        libs_string.push_str(line.as_slice());
    }

    format!(r#"package rust.glutin;

public class MainActivity extends android.app.NativeActivity {{
    static {{
        {0}
    }}
}}"#, libs_string)
}

fn build_manifest(crate_name: &str, activity_name: &str) -> String {
    format!(r#"<?xml version="1.0" encoding="utf-8"?>
<!-- BEGIN_INCLUDE(manifest) -->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
        package="com.example.native_activity"
        android:versionCode="1"
        android:versionName="1.0">

    <uses-sdk android:minSdkVersion="9" />

    <uses-feature android:glEsVersion="0x00020000" android:required="true"></uses-feature>
    <uses-permission android:name="android.permission.INTERNET" />
    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />

    <application android:label="{0}">
        <activity android:name="{1}"
                android:label="{0}"
                android:configChanges="orientation|keyboardHidden">
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />
                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
    </application>

</manifest>
<!-- END_INCLUDE(manifest) -->
"#, crate_name, activity_name)
}

fn build_build_xml() -> String {
    format!(r#"<?xml version="1.0" encoding="UTF-8"?>
<project name="rust-android" default="help">
    <property file="local.properties" />
    <loadproperties srcFile="project.properties" />
    <import file="custom_rules.xml" optional="true" />
    <import file="${{sdk.dir}}/tools/ant/build.xml" />
</project>
"#)
}

fn build_local_properties(sdk_dir: &Path) -> String {
    use std::os;
    format!(r"sdk.dir={}", os::make_absolute(sdk_dir).display())
}

fn build_project_properties() -> String {
    format!(r"target=android-18")
}
<commit_msg>Fix for rustc<commit_after>extern crate serialize;

use std::collections::HashMap;
use std::io::process::Command;
use std::io::{File, TempDir};
use std::io::fs;

fn main() {
    let (args, passthrough) = parse_arguments();

    // Find all the native shared libraries that exist in the target directory.
    let native_shared_libs = find_native_libs(&args);

    // getting the path from the ANDROID_HOME env
    let sdk_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "ANDROID_HOME")
        .map(|(_, v)| Path::new(v)).expect("Please set the ANDROID_HOME environment variable");

    // hardcoding ndk path
    let ndk_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "NDK_HOME")
        .map(|(_, v)| Path::new(v)).expect("Please set the NDK_HOME environment variable");

    // hardcoding ndk path
    let standalone_path = std::os::env().into_iter().find(|&(ref k, _)| k.as_slice() == "NDK_STANDALONE")
        .map(|(_, v)| Path::new(v)).unwrap_or(Path::new("/opt/ndk_standalone"));

    // creating the build directory that will contain all the necessary files to create teh apk
    let directory = build_directory(&sdk_path, args.output.filestem_str().unwrap(), &native_shared_libs);

    // Copy the additional native libs into the libs directory.
    for (name, path) in native_shared_libs.iter() {
        fs::copy(path, &directory.path().join("libs").join("armeabi").join(name)).unwrap();
    }

    // compiling android_native_app_glue.c
    if Command::new(standalone_path.join("bin").join("arm-linux-androideabi-gcc"))
        .arg(ndk_path.join("sources").join("android").join("native_app_glue").join("android_native_app_glue.c"))
        .arg("-c")
        .arg("-o").arg(directory.path().join("android_native_app_glue.o"))
        .stdout(std::io::process::InheritFd(1)).stderr(std::io::process::InheritFd(2))
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing gcc");
        std::os::set_exit_status(1);
        return;
    }

    // calling gcc to link to a shared object
    if Command::new(standalone_path.join("bin").join("arm-linux-androideabi-gcc"))
        .args(passthrough.as_slice())
        .arg(directory.path().join("android_native_app_glue.o"))
        .arg("-o").arg(directory.path().join("libs").join("armeabi").join("libmain.so"))
        .arg("-shared")
        .arg("-Wl,-E")
        .stdout(std::io::process::InheritFd(1))
        .stderr(std::io::process::InheritFd(2))//.cwd(directory.path())
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing gcc");
        std::os::set_exit_status(1);
        return;
    }

    // calling objdump to make sure that our object has `ANativeActivity_onCreate`
    // TODO: not working
    /*{
        let mut process =
            Command::new(standalone_path.join("bin").join("arm-linux-androideabi-objdump"))
            .arg("-x").arg(directory.path().join("libs").join("armeabi").join("libmain.so"))
            .stderr(std::io::process::InheritFd(2))
            .spawn().unwrap();

        // TODO: use UFCS instead
        fn by_ref<'a, T: Reader>(r: &'a mut T) -> std::io::RefReader<'a, T> { r.by_ref() };

        let stdout = process.stdout.as_mut().unwrap();
        let mut stdout = std::io::BufferedReader::new(by_ref(stdout));

        if stdout.lines().filter_map(|l| l.ok())
            .find(|line| line.as_slice().contains("ANativeActivity_onCreate")).is_none()
        {
            println!("Error: the output file doesn't contain ANativeActivity_onCreate");
            std::os::set_exit_status(1);
            return;
        }
    }*/

    // executing ant
    if Command::new("ant").arg("debug").stdout(std::io::process::InheritFd(1))
        .stderr(std::io::process::InheritFd(2)).cwd(directory.path())
        .status().unwrap() != std::io::process::ExitStatus(0)
    {
        println!("Error while executing ant debug");
        std::os::set_exit_status(1);
        return;
    }

    // copying apk file to the requested output
    fs::copy(&directory.path().join("bin").join("rust-android-debug.apk"),
        &args.output).unwrap();
}

struct Args {
    output: Path,
}

fn parse_arguments() -> (Args, Vec<String>) {
    let mut result_output = None;
    let mut result_passthrough = Vec::new();

    let args = std::os::args();
    let mut args = args.into_iter().skip(1);

    loop {
        let arg = match args.next() {
            None => return (
                Args {
                    output: result_output.expect("Could not find -o argument")
                },
                result_passthrough
            ),
            Some(arg) => arg
        };

        match arg.as_slice() {
            "-o" => {
                result_output = Some(Path::new(args.next().expect("-o must be followed by the output name")));
            },
            _ => result_passthrough.push(arg)
        };
    }
}

fn find_native_libs(args: &Args) -> HashMap<String, Path> {
    let base_path = args.output.dir_path().join("native");
    let mut native_shared_libs: HashMap<String, Path> = HashMap::new();

    fs::walk_dir(&base_path).and_then(|mut dirs| {
        for dir in dirs {
            fs::readdir(&dir).and_then(|paths| {
                for path in paths.iter() {
                    match (path.filename_str(), path.extension_str()) {
                        (Some(filename), Some(ext)) => {
                            if filename.starts_with("lib") && ext == "so" {
                                native_shared_libs.insert(filename.to_string(), path.clone());
                            }
                        }
                        _ => {}
                    }
                }
                Ok(())
            }).ok();
        }
        Ok(())
    }).ok();

    native_shared_libs
}

fn build_directory(sdk_dir: &Path, crate_name: &str, libs: &HashMap<String, Path>) -> TempDir {
    use std::io::fs;

    let build_directory = TempDir::new("android-rs-glue-rust-to-apk")
        .ok().expect("Could not create temporary build directory");

    let activity_name = if libs.len() > 0 {
        let src_path = build_directory.path().join("src/rust/glutin");
        fs::mkdir_recursive(&src_path, std::io::USER_RWX).unwrap();

        File::create(&src_path.join("MainActivity.java")).unwrap()
            .write_str(java_src(libs).as_slice())
            .unwrap();

        "rust.glutin.MainActivity"
    } else {
        "android.app.NativeActivity"
    };

    File::create(&build_directory.path().join("AndroidManifest.xml")).unwrap()
        .write_str(build_manifest(crate_name, activity_name).as_slice())
        .unwrap();

    File::create(&build_directory.path().join("build.xml")).unwrap()
        .write_str(build_build_xml().as_slice())
        .unwrap();

    File::create(&build_directory.path().join("local.properties")).unwrap()
        .write_str(build_local_properties(sdk_dir).as_slice())
        .unwrap();

    File::create(&build_directory.path().join("project.properties")).unwrap()
        .write_str(build_project_properties().as_slice())
        .unwrap();

    {
        let libs_path = build_directory.path().join("libs").join("armeabi");
        fs::mkdir_recursive(&libs_path, std::io::USER_RWX).unwrap();
    }

    build_directory
}

fn java_src(libs: &HashMap<String, Path>) -> String {
    let mut libs_string = "".to_string();

    for (name, _) in libs.iter() {
        // Strip off the 'lib' prefix and ".so" suffix. This is safe since libs only get added
        // to the hash map if they start with lib.
        let line = format!("        System.loadLibrary(\"{}\");\n", name.slice(3, name.len()-3));
        libs_string.push_str(line.as_slice());
    }

    format!(r#"package rust.glutin;

public class MainActivity extends android.app.NativeActivity {{
    static {{
        {0}
    }}
}}"#, libs_string)
}

fn build_manifest(crate_name: &str, activity_name: &str) -> String {
    format!(r#"<?xml version="1.0" encoding="utf-8"?>
<!-- BEGIN_INCLUDE(manifest) -->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
        package="com.example.native_activity"
        android:versionCode="1"
        android:versionName="1.0">

    <uses-sdk android:minSdkVersion="9" />

    <uses-feature android:glEsVersion="0x00020000" android:required="true"></uses-feature>
    <uses-permission android:name="android.permission.INTERNET" />
    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />

    <application android:label="{0}">
        <activity android:name="{1}"
                android:label="{0}"
                android:configChanges="orientation|keyboardHidden">
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />
                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
    </application>

</manifest>
<!-- END_INCLUDE(manifest) -->
"#, crate_name, activity_name)
}

fn build_build_xml() -> String {
    format!(r#"<?xml version="1.0" encoding="UTF-8"?>
<project name="rust-android" default="help">
    <property file="local.properties" />
    <loadproperties srcFile="project.properties" />
    <import file="custom_rules.xml" optional="true" />
    <import file="${{sdk.dir}}/tools/ant/build.xml" />
</project>
"#)
}

fn build_local_properties(sdk_dir: &Path) -> String {
    use std::os;
    format!(r"sdk.dir={}", os::make_absolute(sdk_dir).unwrap().display())
}

fn build_project_properties() -> String {
    format!(r"target=android-18")
}
<|endoftext|>


<commit_before>use analysis::ref_mode::RefMode;
use analysis::rust_type::*;
use config;
use env::Env;
use library::{self, Nullable};

#[derive(Clone, Debug, Default)]
pub struct Info {
    pub parameter: Option<library::Parameter>,
    pub base_tid: Option<library::TypeId>,  //Some only if need downcast
    pub commented: bool,
}

pub fn analyze(env: &Env, func: &library::Function, type_tid: library::TypeId,
               configured_functions: &[&config::functions::Function],
               used_types: &mut Vec<String>) -> Info {

    let mut parameter = if func.ret.typ == Default::default() { None } else {
        if let Ok(s) = used_rust_type(env, func.ret.typ) {
            used_types.push(s);
        }
        // Since GIRs are bad at specifying return value nullability, assume
        // any returned pointer is nullable unless overridden by the config.
        let mut nullable = func.ret.nullable;
        if !*nullable && can_be_nullable_return(env, func.ret.typ) {
            *nullable = true;
        }
        let nullable_override = configured_functions.iter()
            .filter_map(|f| f.ret.nullable)
            .next();
        if let Some(val) = nullable_override {
            nullable = val;
        }
        Some(library::Parameter {
                nullable: nullable,
                .. func.ret.clone()
            })
    };

    let commented = if func.ret.typ == Default::default() { false } else {
        parameter_rust_type(env, func.ret.typ, func.ret.direction, Nullable(false), RefMode::None).is_err()
    };

    let mut base_tid = None;

    if func.kind == library::FunctionKind::Constructor {
        if let Some(par) = parameter {
            if par.typ != type_tid {
                base_tid = Some(par.typ);
            }
            parameter = Some(library::Parameter {
                typ: type_tid,
                nullable: Nullable(false),
                ..par
            });
        }
    }

    Info {
        parameter: parameter,
        base_tid: base_tid,
        commented: commented,
    }
}

fn can_be_nullable_return(env: &Env, type_id: library::TypeId) -> bool
{
    use library::Type::*;
    use library::Fundamental::*;
    match *env.library.type_(type_id) {
        Fundamental(fund) => match fund {
            Pointer => true,
            Utf8 => true,
            Filename => true,
            _ => false,
        },
        Alias(ref alias) => can_be_nullable_return(env, alias.typ),
        Enumeration(_) => false,
        Bitfield(_) => false,
        Record(_) => true,
        Union(_) => true,
        Function(_) => true,
        Interface(_) => true,
        Class(_) => true,
        _ => true
    }
}
<commit_msg>Add nullable override for constructors<commit_after>use analysis::ref_mode::RefMode;
use analysis::rust_type::*;
use config;
use env::Env;
use library::{self, Nullable};

#[derive(Clone, Debug, Default)]
pub struct Info {
    pub parameter: Option<library::Parameter>,
    pub base_tid: Option<library::TypeId>,  //Some only if need downcast
    pub commented: bool,
}

pub fn analyze(env: &Env, func: &library::Function, type_tid: library::TypeId,
               configured_functions: &[&config::functions::Function],
               used_types: &mut Vec<String>) -> Info {

    let mut parameter = if func.ret.typ == Default::default() { None } else {
        if let Ok(s) = used_rust_type(env, func.ret.typ) {
            used_types.push(s);
        }
        // Since GIRs are bad at specifying return value nullability, assume
        // any returned pointer is nullable unless overridden by the config.
        let mut nullable = func.ret.nullable;
        if !*nullable && can_be_nullable_return(env, func.ret.typ) {
            *nullable = true;
        }
        let nullable_override = configured_functions.iter()
            .filter_map(|f| f.ret.nullable)
            .next();
        if let Some(val) = nullable_override {
            nullable = val;
        }
        Some(library::Parameter {
                nullable: nullable,
                .. func.ret.clone()
            })
    };

    let commented = if func.ret.typ == Default::default() { false } else {
        parameter_rust_type(env, func.ret.typ, func.ret.direction, Nullable(false), RefMode::None).is_err()
    };

    let mut base_tid = None;

    if func.kind == library::FunctionKind::Constructor {
        if let Some(par) = parameter {
            let nullable_override = configured_functions.iter()
                .filter_map(|f| f.ret.nullable)
                .next();
            if par.typ != type_tid {
                base_tid = Some(par.typ);
            }
            parameter = Some(library::Parameter {
                typ: type_tid,
                nullable: nullable_override.unwrap_or(Nullable(false)),
                ..par
            });
        }
    }

    Info {
        parameter: parameter,
        base_tid: base_tid,
        commented: commented,
    }
}

fn can_be_nullable_return(env: &Env, type_id: library::TypeId) -> bool
{
    use library::Type::*;
    use library::Fundamental::*;
    match *env.library.type_(type_id) {
        Fundamental(fund) => match fund {
            Pointer => true,
            Utf8 => true,
            Filename => true,
            _ => false,
        },
        Alias(ref alias) => can_be_nullable_return(env, alias.typ),
        Enumeration(_) => false,
        Bitfield(_) => false,
        Record(_) => true,
        Union(_) => true,
        Function(_) => true,
        Interface(_) => true,
        Class(_) => true,
        _ => true
    }
}
<|endoftext|>


<commit_before>use inflector::Inflector;
use lazy_static::lazy_static;
use linked_hash_map::LinkedHashMap;
use std::collections::HashSet;

use crate::options::Options;
use crate::shape::{self, Shape};
use crate::util::lower_camel_case;
use crate::util::type_case;

pub struct Ctxt {
    options: Options,
    type_names: HashSet<String>,
}

pub type Ident = String;
pub type Code = String;

pub fn kotlin_types(name: &str, shape: &Shape, options: Options) -> (Ident, Option<Code>) {
    let mut ctxt = Ctxt {
        options,
        type_names: HashSet::new(),
    };

    type_from_shape(&mut ctxt, name, shape)
}

fn type_from_shape(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    use crate::shape::Shape::*;
    match *shape {
        Null | Any | Bottom => ("Any".into(), None),
        Bool => ("Boolean".into(), None),
        StringT => ("String".into(), None),
        Integer => ("Long".into(), None),
        Floating => ("Double".into(), None),
        Tuple(ref shapes, _n) => {
            let folded = shape::fold_shapes(shapes.clone());
            if shapes.len() <= 3 && folded == Any && shapes.iter().any(|s| s != &Any) {
                generate_tuple_type(ctxt, path, shapes)
            } else {
                generate_vec_type(ctxt, path, &folded)
            }
        }
        VecT { elem_type: ref e } => generate_vec_type(ctxt, path, e),
        Struct { fields: ref map } => generate_struct_from_field_shapes(ctxt, path, map),
        MapT { val_type: ref v } => generate_map_type(ctxt, path, v),
        Opaque(ref t) => (t.clone(), None),
        Optional(ref e) => {
            let (inner, defs) = type_from_shape(ctxt, path, e);
            if ctxt.options.use_default_for_missing_fields {
                (inner, defs)
            } else {
                (format!("{}?", inner), defs)
            }
        }
    }
}

fn generate_vec_type(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    let singular = path.to_singular();
    let (inner, defs) = type_from_shape(ctxt, &singular, shape);
    (format!("List<{}>", inner), defs)
}

fn generate_map_type(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    let singular = path.to_singular();
    let (inner, defs) = type_from_shape(ctxt, &singular, shape);
    (format!("Map<String, {}>", inner), defs)
}

fn generate_tuple_type(ctxt: &mut Ctxt, path: &str, shapes: &[Shape]) -> (Ident, Option<Code>) {
    let mut types = Vec::new();
    let mut defs = Vec::new();

    for shape in shapes {
        let (typ, def) = type_from_shape(ctxt, path, shape);
        types.push(typ);
        if let Some(code) = def {
            defs.push(code)
        }
    }

    (
        format!(
            "{}<{}>",
            match types.len() {
                2 => "Pair",
                3 => "Triple",
                _ => panic!("No n-tuple type exists for n of {}", types.len()),
            },
            types.join(", ")
        ),
        Some(defs.join("\n\n")),
    )
}

fn field_name(name: &str, used_names: &HashSet<String>) -> Ident {
    type_or_field_name(name, used_names, "field", lower_camel_case)
}

fn type_name(name: &str, used_names: &HashSet<String>) -> Ident {
    type_or_field_name(name, used_names, "GeneratedType", type_case)
}

// https://kotlinlang.org/docs/reference/keyword-reference.html
const KOTLIN_KEYWORDS_ARR: &[&str] = &[
    // Hard
    "as", "break", "class", "continue", "do", "else", "false", "for", "fun", "if", "in",
    "interface", "is", "null", "object", "package", "return", "super", "this", "throw", "true",
    "try", "typealias", "val", "var", "when", "while",

    // Soft
    "by", "catch", "constructor", "delegate", "dynamic", "field", "file", "finally", "get",
    "import", "init", "param", "property", "receiver", "set", "setparam", "where",

    // Modifier
    "actual", "abstract", "annotation", "companion", "const", "crossinline", "data", "enum",
    "expect", "external", "final", "infix", "inline", "inner", "internal", "lateinit", "noinline",
    "open", "operator", "out", "override", "private", "protected", "public", "reified", "sealed",
    "suspend", "tailrec", "vararg",

    // Special
    "field", "it"
];

lazy_static! {
    static ref KOTLIN_KEYWORDS: HashSet<&'static str> =
        { KOTLIN_KEYWORDS_ARR.iter().cloned().collect() };
}

fn type_or_field_name(
    name: &str,
    used_names: &HashSet<String>,
    default_name: &str,
    case_fn: fn(&str) -> String,
) -> Ident {
    let name = name.trim();
    let mut output_name = if let Some(c) = name.chars().next() {
        if c.is_ascii() && c.is_numeric() {
            let temp = String::from("n") + name;
            case_fn(&temp)
        } else {
            case_fn(name)
        }
    } else {
        case_fn(name)
    };
    if KOTLIN_KEYWORDS.contains::<str>(&output_name) {
        output_name.push_str("_field");
    }
    if output_name == "" {
        output_name.push_str(default_name);
    }
    if !used_names.contains(&output_name) {
        return output_name;
    }
    for n in 2.. {
        let temp = format!("{}{}", output_name, n);
        if !used_names.contains(&temp) {
            return temp;
        }
    }
    unreachable!()
}

fn generate_struct_from_field_shapes(
    ctxt: &mut Ctxt,
    path: &str,
    map: &LinkedHashMap<String, Shape>,
) -> (Ident, Option<Code>) {
    let type_name = type_name(path, &ctxt.type_names);
    ctxt.type_names.insert(type_name.clone());

    let mut field_names = HashSet::new();
    let mut defs = Vec::new();

    let fields: Vec<Code> = map
        .iter()
        .map(|(name, typ)| {
            let field_name = field_name(name, &field_names);
            field_names.insert(field_name.clone());

            let mut field_code = String::new();
            if &field_name != name {
                field_code += &format!("    @field:JsonProperty(\"{}\")]\n", name)
            }

            let (field_type, child_defs) = type_from_shape(ctxt, name, typ);

            if let Some(code) = child_defs {
                defs.push(code);
            }

            format!("{}    val {}: {}", field_code, field_name, field_type)
        })
        .collect();

    let mut code = format!("class {}(\n", type_name);

    if !fields.is_empty() {
        code += &fields.join(",\n");
        code += "\n";
    }
    code += ")";

    if !defs.is_empty() {
        code += "\n\n";
        code += &defs.join("\n\n");
    }

    (type_name, Some(code))
}
<commit_msg>Remove extra bracket in Kotlin codegen.<commit_after>use inflector::Inflector;
use lazy_static::lazy_static;
use linked_hash_map::LinkedHashMap;
use std::collections::HashSet;

use crate::options::Options;
use crate::shape::{self, Shape};
use crate::util::lower_camel_case;
use crate::util::type_case;

pub struct Ctxt {
    options: Options,
    type_names: HashSet<String>,
}

pub type Ident = String;
pub type Code = String;

pub fn kotlin_types(name: &str, shape: &Shape, options: Options) -> (Ident, Option<Code>) {
    let mut ctxt = Ctxt {
        options,
        type_names: HashSet::new(),
    };

    type_from_shape(&mut ctxt, name, shape)
}

fn type_from_shape(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    use crate::shape::Shape::*;
    match *shape {
        Null | Any | Bottom => ("Any".into(), None),
        Bool => ("Boolean".into(), None),
        StringT => ("String".into(), None),
        Integer => ("Long".into(), None),
        Floating => ("Double".into(), None),
        Tuple(ref shapes, _n) => {
            let folded = shape::fold_shapes(shapes.clone());
            if shapes.len() <= 3 && folded == Any && shapes.iter().any(|s| s != &Any) {
                generate_tuple_type(ctxt, path, shapes)
            } else {
                generate_vec_type(ctxt, path, &folded)
            }
        }
        VecT { elem_type: ref e } => generate_vec_type(ctxt, path, e),
        Struct { fields: ref map } => generate_struct_from_field_shapes(ctxt, path, map),
        MapT { val_type: ref v } => generate_map_type(ctxt, path, v),
        Opaque(ref t) => (t.clone(), None),
        Optional(ref e) => {
            let (inner, defs) = type_from_shape(ctxt, path, e);
            if ctxt.options.use_default_for_missing_fields {
                (inner, defs)
            } else {
                (format!("{}?", inner), defs)
            }
        }
    }
}

fn generate_vec_type(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    let singular = path.to_singular();
    let (inner, defs) = type_from_shape(ctxt, &singular, shape);
    (format!("List<{}>", inner), defs)
}

fn generate_map_type(ctxt: &mut Ctxt, path: &str, shape: &Shape) -> (Ident, Option<Code>) {
    let singular = path.to_singular();
    let (inner, defs) = type_from_shape(ctxt, &singular, shape);
    (format!("Map<String, {}>", inner), defs)
}

fn generate_tuple_type(ctxt: &mut Ctxt, path: &str, shapes: &[Shape]) -> (Ident, Option<Code>) {
    let mut types = Vec::new();
    let mut defs = Vec::new();

    for shape in shapes {
        let (typ, def) = type_from_shape(ctxt, path, shape);
        types.push(typ);
        if let Some(code) = def {
            defs.push(code)
        }
    }

    (
        format!(
            "{}<{}>",
            match types.len() {
                2 => "Pair",
                3 => "Triple",
                _ => panic!("No n-tuple type exists for n of {}", types.len()),
            },
            types.join(", ")
        ),
        Some(defs.join("\n\n")),
    )
}

fn field_name(name: &str, used_names: &HashSet<String>) -> Ident {
    type_or_field_name(name, used_names, "field", lower_camel_case)
}

fn type_name(name: &str, used_names: &HashSet<String>) -> Ident {
    type_or_field_name(name, used_names, "GeneratedType", type_case)
}

// https://kotlinlang.org/docs/reference/keyword-reference.html
const KOTLIN_KEYWORDS_ARR: &[&str] = &[
    // Hard
    "as", "break", "class", "continue", "do", "else", "false", "for", "fun", "if", "in",
    "interface", "is", "null", "object", "package", "return", "super", "this", "throw", "true",
    "try", "typealias", "val", "var", "when", "while",

    // Soft
    "by", "catch", "constructor", "delegate", "dynamic", "field", "file", "finally", "get",
    "import", "init", "param", "property", "receiver", "set", "setparam", "where",

    // Modifier
    "actual", "abstract", "annotation", "companion", "const", "crossinline", "data", "enum",
    "expect", "external", "final", "infix", "inline", "inner", "internal", "lateinit", "noinline",
    "open", "operator", "out", "override", "private", "protected", "public", "reified", "sealed",
    "suspend", "tailrec", "vararg",

    // Special
    "field", "it"
];

lazy_static! {
    static ref KOTLIN_KEYWORDS: HashSet<&'static str> =
        { KOTLIN_KEYWORDS_ARR.iter().cloned().collect() };
}

fn type_or_field_name(
    name: &str,
    used_names: &HashSet<String>,
    default_name: &str,
    case_fn: fn(&str) -> String,
) -> Ident {
    let name = name.trim();
    let mut output_name = if let Some(c) = name.chars().next() {
        if c.is_ascii() && c.is_numeric() {
            let temp = String::from("n") + name;
            case_fn(&temp)
        } else {
            case_fn(name)
        }
    } else {
        case_fn(name)
    };
    if KOTLIN_KEYWORDS.contains::<str>(&output_name) {
        output_name.push_str("_field");
    }
    if output_name == "" {
        output_name.push_str(default_name);
    }
    if !used_names.contains(&output_name) {
        return output_name;
    }
    for n in 2.. {
        let temp = format!("{}{}", output_name, n);
        if !used_names.contains(&temp) {
            return temp;
        }
    }
    unreachable!()
}

fn generate_struct_from_field_shapes(
    ctxt: &mut Ctxt,
    path: &str,
    map: &LinkedHashMap<String, Shape>,
) -> (Ident, Option<Code>) {
    let type_name = type_name(path, &ctxt.type_names);
    ctxt.type_names.insert(type_name.clone());

    let mut field_names = HashSet::new();
    let mut defs = Vec::new();

    let fields: Vec<Code> = map
        .iter()
        .map(|(name, typ)| {
            let field_name = field_name(name, &field_names);
            field_names.insert(field_name.clone());

            let mut field_code = String::new();
            if &field_name != name {
                field_code += &format!("    @field:JsonProperty(\"{}\")\n", name)
            }

            let (field_type, child_defs) = type_from_shape(ctxt, name, typ);

            if let Some(code) = child_defs {
                defs.push(code);
            }

            format!("{}    val {}: {}", field_code, field_name, field_type)
        })
        .collect();

    let mut code = format!("class {}(\n", type_name);

    if !fields.is_empty() {
        code += &fields.join(",\n");
        code += "\n";
    }
    code += ")";

    if !defs.is_empty() {
        code += "\n\n";
        code += &defs.join("\n\n");
    }

    (type_name, Some(code))
}
<|endoftext|>


<commit_before>use crate::inference::jsoninputerr::JsonInputErr;
use std::io::{BufReader, Bytes, Read};
use std::iter::Peekable;

#[derive(Debug, PartialEq)]
pub enum JsonToken {
    True,
    False,
    Null,
    Number(String),
    String(String),
    ObjectStart,
    ObjectEnd,
    ArrayStart,
    ArrayEnd,
    Comma,
    Colon,
}

pub struct JsonLexer<R: Read> {
    bytes: Peekable<Bytes<BufReader<R>>>, // TODO: Keep position info
    failed: bool,
    buffer: Vec<u8>,
}

impl<R: Read> JsonLexer<R> {
    pub fn new(source: R) -> Self {
        JsonLexer {
            bytes: BufReader::new(source).bytes().peekable(),
            failed: false,
            buffer: Vec::new(),
        }
    }

    fn get_next_token(&mut self) -> Option<Result<JsonToken, JsonInputErr>> {
        loop {
            let byte = match self.bytes.peek() {
                Some(Ok(byte)) => *byte,
                Some(Err(_)) => return Some(Err(JsonInputErr::IoErr)),
                None => return None,
            };

            return Some(match byte {
                b' ' | b'\t' | b'\n' | b'\r' => {
                    self.bytes.next();
                    continue;
                }
                b'{' => self.skip_and_produce(JsonToken::ObjectStart),
                b'}' => self.skip_and_produce(JsonToken::ObjectEnd),
                b'[' => self.skip_and_produce(JsonToken::ArrayStart),
                b']' => self.skip_and_produce(JsonToken::ArrayEnd),
                b',' => self.skip_and_produce(JsonToken::Comma),
                b':' => self.skip_and_produce(JsonToken::Colon),
                b't' => self.match_token("true", JsonToken::True),
                b'f' => self.match_token("false", JsonToken::False),
                b'n' => self.match_token("null", JsonToken::Null),
                b'"' => self.match_string(),
                b'0'..=b'9' | b'-' => self.match_number(),
                _ => Err(JsonInputErr::InvalidJson),
            });
        }
    }

    fn skip_and_produce(&mut self, token: JsonToken) -> Result<JsonToken, JsonInputErr> {
        self.bytes.next();
        Ok(token)
    }

    fn expect_byte(&mut self) -> Result<u8, JsonInputErr> {
        match self.bytes.next() {
            Some(Ok(byte)) => Ok(byte),
            Some(Err(_)) => Err(JsonInputErr::IoErr),
            None => Err(JsonInputErr::UnexpectedEndOfInput),
        }
    }

    fn skip_byte(&mut self, target_byte: u8) -> Result<(), JsonInputErr> {
        let byte = self.expect_byte()?;
        if byte == target_byte {
            Ok(())
        } else {
            Err(JsonInputErr::InvalidJson)
        }
    }

    fn match_token(
        &mut self,
        target_str: &'static str,
        token: JsonToken,
    ) -> Result<JsonToken, JsonInputErr> {
        for target_byte in target_str.bytes() {
            self.skip_byte(target_byte)?;
        }
        Ok(token)
    }

    fn match_string(&mut self) -> Result<JsonToken, JsonInputErr> {
        self.skip_byte(b'"')?;
        self.buffer.clear();
        loop {
            let byte = self.expect_byte()?;

            if byte == b'"' {
                return Ok(JsonToken::String(
                    String::from_utf8(self.buffer.clone())
                        .map_err(|_| JsonInputErr::InvalidUtf8)?,
                ));
            } else if byte == b'\\' {
                let escaped = self.expect_byte()?;

                match escaped {
                    b'"' => self.buffer.push(b'"'),
                    b'\\' => self.buffer.push(b'\\'),
                    b'/' => self.buffer.push(b'/'),
                    b'b' => self.buffer.push(8),  // backspace
                    b'f' => self.buffer.push(12), // form feed
                    b'n' => self.buffer.push(b'\n'),
                    b'r' => self.buffer.push(b'\r'),
                    b't' => self.buffer.push(b'\t'),
                    b'u' => {
                        let surrogate_offset: u32 = (0xD800 << 10) + 0xDC00 - 0x10000;

                        let mut codepoint = self.parse_codepoint()? as u32;
                        if (0xD800..=0xDFFF).contains(&codepoint) {
                            // first codepoint was the start of a surrogate pair
                            self.skip_byte(b'\\')?;
                            self.skip_byte(b'u')?;
                            let codepoint2 = self.parse_codepoint()? as u32;
                            codepoint = ((codepoint << 10) + codepoint2) - surrogate_offset;
                        };
                        let mut buf = [0u8; 4];
                        let encoded_bytes = std::char::from_u32(codepoint)
                            .unwrap_or(std::char::REPLACEMENT_CHARACTER)
                            .encode_utf8(&mut buf)
                            .bytes();
                        for encoded_byte in encoded_bytes {
                            self.buffer.push(encoded_byte);
                        }
                    }
                    _ => return Err(JsonInputErr::InvalidEscape(escaped)),
                };
            } else {
                self.buffer.push(byte)
            }
        }
    }

    // "ab03..." -> 0xab03
    fn parse_codepoint(&mut self) -> Result<u16, JsonInputErr> {
        let mut codepoint: u16 = 0;
        for _ in 0..4 {
            codepoint <<= 4;
            let byte2 = self.expect_byte()?;
            codepoint += match byte2 {
                b'0'..=b'9' => byte2 - b'0',
                b'a'..=b'f' => byte2 - b'a' + 10,
                b'A'..=b'F' => byte2 - b'A' + 10,
                _ => return Err(JsonInputErr::InvalidEscape(byte2)),
            } as u16;
        }
        Ok(codepoint)
    }

    fn match_number(&mut self) -> Result<JsonToken, JsonInputErr> {
        let mut buffer = Vec::new();
        loop {
            let byte = match self.bytes.peek() {
                Some(Ok(byte)) => *byte,
                Some(Err(_)) => return Err(JsonInputErr::IoErr),
                None => break,
            };

            match byte {
                b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E' => {
                    buffer.push(byte);
                    self.bytes.next();
                }
                _ => break,
            }
        }
        // TODO: Actually parse numbers
        Ok(JsonToken::Number(
            String::from_utf8(buffer).map_err(|_err| JsonInputErr::InvalidUtf8)?,
        ))
    }
}

impl<R: Read> Iterator for JsonLexer<R> {
    type Item = Result<JsonToken, JsonInputErr>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.failed {
            return None;
        }

        let res = self.get_next_token();
        if let Some(Err(_)) = res {
            self.failed = true;
        }
        res
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::inference::jsoninputerr::JsonInputErr;
    use std::fmt::Debug;

    #[test]
    fn empty_input() {
        assert_eq!(tokens_from_str(""), Ok(vec![]));
        assert_eq!(tokens_from_str(" \t\r\n"), Ok(vec![]));
    }

    #[test]
    fn bare_number() {
        assert_eq!(
            tokens_from_str("123"),
            Ok(vec![JsonToken::Number("123".to_string())])
        );
    }

    #[test]
    fn object() {
        assert_eq!(
            tokens_from_str("{}"),
            Ok(vec![JsonToken::ObjectStart, JsonToken::ObjectEnd])
        );
        assert_eq!(
            tokens_from_str("  {  }  "),
            Ok(vec![JsonToken::ObjectStart, JsonToken::ObjectEnd])
        );
    }

    #[test]
    fn string() {
        assert_eq!(
            tokens_from_str(r#" "hello world" "#),
            Ok(vec![JsonToken::String("hello world".to_string())])
        );
    }

    #[test]
    fn escapes() {
        assert_eq!(
            tokens_from_str(r#" "foo\nbar" "#),
            Ok(vec![JsonToken::String("foo\nbar".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "John says \"Hello\"" "#),
            Ok(vec![JsonToken::String(r#"John says "Hello""#.to_string())])
        );
    }

    #[test]
    fn unicode_escapes() {
        assert_eq!(
            tokens_from_str(r#" "\u00e6" "#),
            Ok(vec![JsonToken::String("æ".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "\uD83D\uDE00" "#),
            Ok(vec![JsonToken::String("😀".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "\uD83D" "#),
            Err(JsonInputErr::InvalidJson)
        );
    }

    #[test]
    fn number() {
        assert_eq!(
            tokens_from_str(r#" 14.5 "#),
            Ok(vec![JsonToken::Number("14.5".to_string())])
        );
    }

    #[test]
    fn complex() {
        assert_eq!(
            tokens_from_str(
                r#"
                {
                    "foo": [1, true]
                }
                "#
            ),
            Ok(vec![
                JsonToken::ObjectStart,
                JsonToken::String("foo".to_string()),
                JsonToken::Colon,
                JsonToken::ArrayStart,
                JsonToken::Number("1".to_string()),
                JsonToken::Comma,
                JsonToken::True,
                JsonToken::ArrayEnd,
                JsonToken::ObjectEnd
            ])
        );
    }

    #[test]
    fn invalid() {
        assert_eq!(tokens_from_str("foo"), Err(JsonInputErr::InvalidJson));
        assert_eq!(tokens_from_str(" [ foo ] "), Err(JsonInputErr::InvalidJson));
    }

    fn tokens_from_str(s: &'static str) -> Result<Vec<JsonToken>, JsonInputErr> {
        let collected: Vec<Result<JsonToken, JsonInputErr>> =
            JsonLexer::new(s.as_bytes()).collect();
        coalesce_err(collected)
    }

    fn coalesce_err<T: Debug, E: Debug>(vec: Vec<Result<T, E>>) -> Result<Vec<T>, E> {
        let error_count = vec.iter().filter(|res| res.is_err()).count();
        match error_count {
            0 => Ok(vec.into_iter().map(|res| res.unwrap()).collect()),
            1 => Err(vec
                .into_iter()
                .find(|res| res.is_err())
                .unwrap()
                .unwrap_err()),
            _ => panic!("More than one error: {:?}", vec),
        }
    }
}
<commit_msg>perf, lexer: Use 128K buffer for BufReader<commit_after>use crate::inference::jsoninputerr::JsonInputErr;
use std::io::{BufReader, Bytes, Read};
use std::iter::Peekable;

#[derive(Debug, PartialEq)]
pub enum JsonToken {
    True,
    False,
    Null,
    Number(String),
    String(String),
    ObjectStart,
    ObjectEnd,
    ArrayStart,
    ArrayEnd,
    Comma,
    Colon,
}

pub struct JsonLexer<R: Read> {
    bytes: Peekable<Bytes<BufReader<R>>>, // TODO: Keep position info
    failed: bool,
    buffer: Vec<u8>,
}

impl<R: Read> JsonLexer<R> {
    pub fn new(source: R) -> Self {
        JsonLexer {
            bytes: BufReader::with_capacity(128 * 1024, source)
                .bytes()
                .peekable(),
            failed: false,
            buffer: Vec::new(),
        }
    }

    fn get_next_token(&mut self) -> Option<Result<JsonToken, JsonInputErr>> {
        loop {
            let byte = match self.bytes.peek() {
                Some(Ok(byte)) => *byte,
                Some(Err(_)) => return Some(Err(JsonInputErr::IoErr)),
                None => return None,
            };

            return Some(match byte {
                b' ' | b'\t' | b'\n' | b'\r' => {
                    self.bytes.next();
                    continue;
                }
                b'{' => self.skip_and_produce(JsonToken::ObjectStart),
                b'}' => self.skip_and_produce(JsonToken::ObjectEnd),
                b'[' => self.skip_and_produce(JsonToken::ArrayStart),
                b']' => self.skip_and_produce(JsonToken::ArrayEnd),
                b',' => self.skip_and_produce(JsonToken::Comma),
                b':' => self.skip_and_produce(JsonToken::Colon),
                b't' => self.match_token("true", JsonToken::True),
                b'f' => self.match_token("false", JsonToken::False),
                b'n' => self.match_token("null", JsonToken::Null),
                b'"' => self.match_string(),
                b'0'..=b'9' | b'-' => self.match_number(),
                _ => Err(JsonInputErr::InvalidJson),
            });
        }
    }

    fn skip_and_produce(&mut self, token: JsonToken) -> Result<JsonToken, JsonInputErr> {
        self.bytes.next();
        Ok(token)
    }

    fn expect_byte(&mut self) -> Result<u8, JsonInputErr> {
        match self.bytes.next() {
            Some(Ok(byte)) => Ok(byte),
            Some(Err(_)) => Err(JsonInputErr::IoErr),
            None => Err(JsonInputErr::UnexpectedEndOfInput),
        }
    }

    fn skip_byte(&mut self, target_byte: u8) -> Result<(), JsonInputErr> {
        let byte = self.expect_byte()?;
        if byte == target_byte {
            Ok(())
        } else {
            Err(JsonInputErr::InvalidJson)
        }
    }

    fn match_token(
        &mut self,
        target_str: &'static str,
        token: JsonToken,
    ) -> Result<JsonToken, JsonInputErr> {
        for target_byte in target_str.bytes() {
            self.skip_byte(target_byte)?;
        }
        Ok(token)
    }

    fn match_string(&mut self) -> Result<JsonToken, JsonInputErr> {
        self.skip_byte(b'"')?;
        self.buffer.clear();
        loop {
            let byte = self.expect_byte()?;

            if byte == b'"' {
                return Ok(JsonToken::String(
                    String::from_utf8(self.buffer.clone())
                        .map_err(|_| JsonInputErr::InvalidUtf8)?,
                ));
            } else if byte == b'\\' {
                let escaped = self.expect_byte()?;

                match escaped {
                    b'"' => self.buffer.push(b'"'),
                    b'\\' => self.buffer.push(b'\\'),
                    b'/' => self.buffer.push(b'/'),
                    b'b' => self.buffer.push(8),  // backspace
                    b'f' => self.buffer.push(12), // form feed
                    b'n' => self.buffer.push(b'\n'),
                    b'r' => self.buffer.push(b'\r'),
                    b't' => self.buffer.push(b'\t'),
                    b'u' => {
                        let surrogate_offset: u32 = (0xD800 << 10) + 0xDC00 - 0x10000;

                        let mut codepoint = self.parse_codepoint()? as u32;
                        if (0xD800..=0xDFFF).contains(&codepoint) {
                            // first codepoint was the start of a surrogate pair
                            self.skip_byte(b'\\')?;
                            self.skip_byte(b'u')?;
                            let codepoint2 = self.parse_codepoint()? as u32;
                            codepoint = ((codepoint << 10) + codepoint2) - surrogate_offset;
                        };
                        let mut buf = [0u8; 4];
                        let encoded_bytes = std::char::from_u32(codepoint)
                            .unwrap_or(std::char::REPLACEMENT_CHARACTER)
                            .encode_utf8(&mut buf)
                            .bytes();
                        for encoded_byte in encoded_bytes {
                            self.buffer.push(encoded_byte);
                        }
                    }
                    _ => return Err(JsonInputErr::InvalidEscape(escaped)),
                };
            } else {
                self.buffer.push(byte)
            }
        }
    }

    // "ab03..." -> 0xab03
    fn parse_codepoint(&mut self) -> Result<u16, JsonInputErr> {
        let mut codepoint: u16 = 0;
        for _ in 0..4 {
            codepoint <<= 4;
            let byte2 = self.expect_byte()?;
            codepoint += match byte2 {
                b'0'..=b'9' => byte2 - b'0',
                b'a'..=b'f' => byte2 - b'a' + 10,
                b'A'..=b'F' => byte2 - b'A' + 10,
                _ => return Err(JsonInputErr::InvalidEscape(byte2)),
            } as u16;
        }
        Ok(codepoint)
    }

    fn match_number(&mut self) -> Result<JsonToken, JsonInputErr> {
        let mut buffer = Vec::new();
        loop {
            let byte = match self.bytes.peek() {
                Some(Ok(byte)) => *byte,
                Some(Err(_)) => return Err(JsonInputErr::IoErr),
                None => break,
            };

            match byte {
                b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E' => {
                    buffer.push(byte);
                    self.bytes.next();
                }
                _ => break,
            }
        }
        // TODO: Actually parse numbers
        Ok(JsonToken::Number(
            String::from_utf8(buffer).map_err(|_err| JsonInputErr::InvalidUtf8)?,
        ))
    }
}

impl<R: Read> Iterator for JsonLexer<R> {
    type Item = Result<JsonToken, JsonInputErr>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.failed {
            return None;
        }

        let res = self.get_next_token();
        if let Some(Err(_)) = res {
            self.failed = true;
        }
        res
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::inference::jsoninputerr::JsonInputErr;
    use std::fmt::Debug;

    #[test]
    fn empty_input() {
        assert_eq!(tokens_from_str(""), Ok(vec![]));
        assert_eq!(tokens_from_str(" \t\r\n"), Ok(vec![]));
    }

    #[test]
    fn bare_number() {
        assert_eq!(
            tokens_from_str("123"),
            Ok(vec![JsonToken::Number("123".to_string())])
        );
    }

    #[test]
    fn object() {
        assert_eq!(
            tokens_from_str("{}"),
            Ok(vec![JsonToken::ObjectStart, JsonToken::ObjectEnd])
        );
        assert_eq!(
            tokens_from_str("  {  }  "),
            Ok(vec![JsonToken::ObjectStart, JsonToken::ObjectEnd])
        );
    }

    #[test]
    fn string() {
        assert_eq!(
            tokens_from_str(r#" "hello world" "#),
            Ok(vec![JsonToken::String("hello world".to_string())])
        );
    }

    #[test]
    fn escapes() {
        assert_eq!(
            tokens_from_str(r#" "foo\nbar" "#),
            Ok(vec![JsonToken::String("foo\nbar".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "John says \"Hello\"" "#),
            Ok(vec![JsonToken::String(r#"John says "Hello""#.to_string())])
        );
    }

    #[test]
    fn unicode_escapes() {
        assert_eq!(
            tokens_from_str(r#" "\u00e6" "#),
            Ok(vec![JsonToken::String("æ".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "\uD83D\uDE00" "#),
            Ok(vec![JsonToken::String("😀".to_string())])
        );

        assert_eq!(
            tokens_from_str(r#" "\uD83D" "#),
            Err(JsonInputErr::InvalidJson)
        );
    }

    #[test]
    fn number() {
        assert_eq!(
            tokens_from_str(r#" 14.5 "#),
            Ok(vec![JsonToken::Number("14.5".to_string())])
        );
    }

    #[test]
    fn complex() {
        assert_eq!(
            tokens_from_str(
                r#"
                {
                    "foo": [1, true]
                }
                "#
            ),
            Ok(vec![
                JsonToken::ObjectStart,
                JsonToken::String("foo".to_string()),
                JsonToken::Colon,
                JsonToken::ArrayStart,
                JsonToken::Number("1".to_string()),
                JsonToken::Comma,
                JsonToken::True,
                JsonToken::ArrayEnd,
                JsonToken::ObjectEnd
            ])
        );
    }

    #[test]
    fn invalid() {
        assert_eq!(tokens_from_str("foo"), Err(JsonInputErr::InvalidJson));
        assert_eq!(tokens_from_str(" [ foo ] "), Err(JsonInputErr::InvalidJson));
    }

    fn tokens_from_str(s: &'static str) -> Result<Vec<JsonToken>, JsonInputErr> {
        let collected: Vec<Result<JsonToken, JsonInputErr>> =
            JsonLexer::new(s.as_bytes()).collect();
        coalesce_err(collected)
    }

    fn coalesce_err<T: Debug, E: Debug>(vec: Vec<Result<T, E>>) -> Result<Vec<T>, E> {
        let error_count = vec.iter().filter(|res| res.is_err()).count();
        match error_count {
            0 => Ok(vec.into_iter().map(|res| res.unwrap()).collect()),
            1 => Err(vec
                .into_iter()
                .find(|res| res.is_err())
                .unwrap()
                .unwrap_err()),
            _ => panic!("More than one error: {:?}", vec),
        }
    }
}
<|endoftext|>


<commit_before>use super::Projection;
use geometry::{Span, Space, RegularSpace};
use geometry::dimensions::{Continuous, Partitioned};
use ndarray::{Axis, ArrayView, Array1, Array2};
use utils::cartesian_product;

#[derive(Serialize, Deserialize)]
pub struct RBFNetwork {
    mu: Array2<f64>,
    beta: Array1<f64>,
}

impl RBFNetwork {
    pub fn new(mu: Array2<f64>, sigma: Array1<f64>) -> Self {
        if mu.cols() != sigma.len() {
            panic!("Dimensions of mu ({:?}) and sigma ({:?}) must agree.",
                   mu.shape(),
                   sigma.shape());
        }

        RBFNetwork {
            mu: mu,
            beta: 0.5 / sigma.map(|v| v*v),
        }
    }

    pub fn from_space(input_space: RegularSpace<Partitioned>) -> Self {
        let n_features = match input_space.span() {
            Span::Finite(s) => s,
            _ => { panic!("`RBFNetwork` projection only supports partitioned input spaces.") }
        };

        let centres = input_space.centres();
        let flat_combs = cartesian_product(&centres)
            .iter()
            .cloned()
            .flat_map(|e| e)
            .collect();

        let mu = Array2::from_shape_vec((n_features, input_space.dim()), flat_combs).unwrap();
        let sigma = input_space.iter().map(|d| d.partition_width()).collect();

        RBFNetwork::new(mu, sigma)
    }

    pub fn kernel(&self, input: &[f64]) -> Array1<f64> {
        let d = &self.mu - &ArrayView::from_shape((1, self.mu.cols()), input).unwrap();

        (&d*&d*&self.beta).mapv(|v| (-v.abs()).exp()).sum_axis(Axis(1))
    }
}

impl Projection<RegularSpace<Continuous>> for RBFNetwork {
    fn project_onto(&self, input: &Vec<f64>, phi: &mut Array1<f64>) {
        let p = self.kernel(input);

        phi.scaled_add(1.0/p.scalar_sum(), &p)
    }

    fn dim(&self) -> usize {
        self.mu.cols()
    }

    fn size(&self) -> usize {
        self.mu.rows()
    }

    fn equivalent(&self, other: &Self) -> bool {
        self.mu == other.mu && self.beta == other.beta && self.size() == other.size()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ndarray::{arr1, arr2};

    #[test]
    fn test_size() {
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25])).size(), 1);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25])).size(), 3);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]; 10]), arr1(&[0.25])).size(), 10);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]; 100]), arr1(&[0.25])).size(), 100);
    }

    #[test]
    fn test_dimensionality() {
        assert_eq!(RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25])).dim(), 1);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0, 0.5, 1.0]; 10]), arr1(&[0.1, 0.2, 0.3])).dim(), 3);
    }

    #[test]
    fn test_kernel_relevance() {
        let rbf = RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25]));
        let mut phi = rbf.kernel(&vec![0.0]);

        for i in 1..10 {
            let phi_new = rbf.kernel(&vec![i as f64 / 10.0]);
            assert!(phi_new[0] < phi[0]);

            phi = phi_new
        }
    }

    #[test]
    fn test_kernel_isotropy() {
        let rbf = RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25]));
        let phi = rbf.kernel(&vec![0.0]);

        for i in 1..10 {
            let phi_left = rbf.kernel(&vec![-i as f64 / 10.0]);
            let phi_right = rbf.kernel(&vec![i as f64 / 10.0]);

            assert!(phi_left[0] < phi[0]);
            assert!(phi_right[0] < phi[0]);
            assert_eq!(phi_left, phi_right);
        }
    }

    #[test]
    fn test_projection_1d() {
        let rbf = RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25]));
        let phi = rbf.project(&vec![0.25]);

        assert!(phi.all_close(&arr1(&[0.49546264, 0.49546264, 0.00907471]), 1e-6));
        assert_eq!(phi.scalar_sum(), 1.0);
    }

    #[test]
    fn test_projection_2d() {
        let rbf = RBFNetwork::new(arr2(&[[0.0, -10.0], [0.5, -8.0], [1.0, -6.0]]),
                                  arr1(&[0.25, 2.0]));
        let phi = rbf.project(&vec![0.67, -7.0]);

        assert!(phi.all_close(&arr1(&[0.10579518, 0.50344131, 0.3907635]), 1e-6));
        assert_eq!(phi.scalar_sum(), 1.0);
    }
}
<commit_msg>chore(fa): replace `scalar_sum` for `l1` in RBFNetwork<commit_after>use super::Projection;
use geometry::{Span, Space, RegularSpace};
use geometry::norms::l1;
use geometry::dimensions::{Continuous, Partitioned};
use ndarray::{Axis, ArrayView, Array1, Array2};
use utils::cartesian_product;

#[derive(Clone, Serialize, Deserialize)]
pub struct RBFNetwork {
    mu: Array2<f64>,
    beta: Array1<f64>,
}

impl RBFNetwork {
    pub fn new(mu: Array2<f64>, sigma: Array1<f64>) -> Self {
        if mu.cols() != sigma.len() {
            panic!("Dimensions of mu ({:?}) and sigma ({:?}) must agree.",
                   mu.shape(),
                   sigma.shape());
        }

        RBFNetwork {
            mu: mu,
            beta: 0.5 / sigma.map(|v| v*v),
        }
    }

    pub fn from_space(input_space: RegularSpace<Partitioned>) -> Self {
        let n_features = match input_space.span() {
            Span::Finite(s) => s,
            _ => { panic!("`RBFNetwork` projection only supports partitioned input spaces.") }
        };

        let centres = input_space.centres();
        let flat_combs = cartesian_product(&centres)
            .iter()
            .cloned()
            .flat_map(|e| e)
            .collect();

        let mu = Array2::from_shape_vec((n_features, input_space.dim()), flat_combs).unwrap();
        let sigma = input_space.iter().map(|d| d.partition_width()).collect();

        RBFNetwork::new(mu, sigma)
    }

    pub fn kernel(&self, input: &[f64]) -> Array1<f64> {
        let d = &self.mu - &ArrayView::from_shape((1, self.mu.cols()), input).unwrap();

        (&d*&d*&self.beta).mapv(|v| (-v.abs()).exp()).sum_axis(Axis(1))
    }
}

impl Projection<RegularSpace<Continuous>> for RBFNetwork {
    fn project_onto(&self, input: &Vec<f64>, phi: &mut Array1<f64>) {
        let p = self.kernel(input);
        let z = l1(p.as_slice().unwrap());

        phi.scaled_add(1.0/z, &p);
    }

    fn dim(&self) -> usize {
        self.mu.cols()
    }

    fn size(&self) -> usize {
        self.mu.rows()
    }

    fn equivalent(&self, other: &Self) -> bool {
        self.mu == other.mu && self.beta == other.beta && self.size() == other.size()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ndarray::{arr1, arr2};

    #[test]
    fn test_size() {
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25])).size(), 1);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25])).size(), 3);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]; 10]), arr1(&[0.25])).size(), 10);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0]; 100]), arr1(&[0.25])).size(), 100);
    }

    #[test]
    fn test_dimensionality() {
        assert_eq!(RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25])).dim(), 1);
        assert_eq!(RBFNetwork::new(arr2(&[[0.0, 0.5, 1.0]; 10]), arr1(&[0.1, 0.2, 0.3])).dim(), 3);
    }

    #[test]
    fn test_kernel_relevance() {
        let rbf = RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25]));
        let mut phi = rbf.kernel(&vec![0.0]);

        for i in 1..10 {
            let phi_new = rbf.kernel(&vec![i as f64 / 10.0]);
            assert!(phi_new[0] < phi[0]);

            phi = phi_new
        }
    }

    #[test]
    fn test_kernel_isotropy() {
        let rbf = RBFNetwork::new(arr2(&[[0.0]]), arr1(&[0.25]));
        let phi = rbf.kernel(&vec![0.0]);

        for i in 1..10 {
            let phi_left = rbf.kernel(&vec![-i as f64 / 10.0]);
            let phi_right = rbf.kernel(&vec![i as f64 / 10.0]);

            assert!(phi_left[0] < phi[0]);
            assert!(phi_right[0] < phi[0]);
            assert_eq!(phi_left, phi_right);
        }
    }

    #[test]
    fn test_projection_1d() {
        let rbf = RBFNetwork::new(arr2(&[[0.0], [0.5], [1.0]]), arr1(&[0.25]));
        let phi = rbf.project(&vec![0.25]);

        assert!(phi.all_close(&arr1(&[0.49546264, 0.49546264, 0.00907471]), 1e-6));
        assert_eq!(phi.scalar_sum(), 1.0);
    }

    #[test]
    fn test_projection_2d() {
        let rbf = RBFNetwork::new(arr2(&[[0.0, -10.0], [0.5, -8.0], [1.0, -6.0]]),
                                  arr1(&[0.25, 2.0]));
        let phi = rbf.project(&vec![0.67, -7.0]);

        assert!(phi.all_close(&arr1(&[0.10579518, 0.50344131, 0.3907635]), 1e-6));
        assert_eq!(phi.scalar_sum(), 1.0);
    }
}
<|endoftext|>


<commit_before>use std::rc::Rc;
use std::cell::RefCell;
use std::net::SocketAddr;
use std::collections::HashMap;
use rand::random;
use mio::net::TcpStream;

use network::{AppId,Backend,ConnectionError};

pub struct BackendMap {
  pub instances:    HashMap<AppId, BackendList>,
  pub max_failures: usize,
}

impl BackendMap {
  pub fn new() -> BackendMap {
    BackendMap {
      instances:    HashMap::new(),
      max_failures: 3,
    }
  }

  pub fn add_instance(&mut self, app_id: &str, instance_address: &SocketAddr) {
    self.instances.entry(app_id.to_string()).or_insert(BackendList::new()).add_instance(instance_address);
  }

  pub fn remove_instance(&mut self, app_id: &str, instance_address: &SocketAddr) {
    if let Some(instances) = self.instances.get_mut(app_id) {
      instances.remove_instance(instance_address);
    } else {
      error!("Instance was already removed");
    }
  }

  pub fn close_backend_connection(&mut self, app_id: &str, addr: &SocketAddr) {
    if let Some(app_instances) = self.instances.get_mut(app_id) {
      if let Some(ref mut backend) = app_instances.find_instance(addr) {
        (*backend.borrow_mut()).dec_connections();
      }
    }
  }

  pub fn backend_from_app_id(&mut self, app_id: &str) -> Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError> {
    if let Some(ref mut app_instances) = self.instances.get_mut(app_id) {
      if app_instances.instances.len() == 0 {
        return Err(ConnectionError::NoBackendAvailable);
      }

      for _ in 0..self.max_failures {
        //FIXME: it's probably pretty wasteful to refilter every time here
        let mut instances:Vec<&mut Rc<RefCell<Backend>>> = app_instances.available_instances();
        if instances.is_empty() {
          error!("no more available backends for app {}", app_id);
          return Err(ConnectionError::NoBackendAvailable);
        }
        let rnd = random::<usize>();
        let idx = rnd % instances.len();

        let conn = instances.get_mut(idx).ok_or(ConnectionError::NoBackendAvailable).and_then(|ref mut b| {
          let ref mut backend = *b.borrow_mut();
          info!("Connecting {} -> {:?}", app_id, (backend.address, backend.active_connections, backend.failures));
          let conn = backend.try_connect();
          if backend.failures >= max_failures_per_backend {
            error!("backend {:?} connections failed {} times, disabling it", (backend.address, backend.active_connections), backend.failures);
          }

          conn.map(|c| (b.clone(), c))
        });

        if conn.is_ok() {
          return conn;
        }
      }
      Err(ConnectionError::NoBackendAvailable)
    } else {
      Err(ConnectionError::NoBackendAvailable)
    }
  }

  pub fn backend_from_sticky_session(&mut self, app_id: &str, sticky_session: u32) -> Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError> {
    let sticky_conn: Option<Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError>> = self.instances
      .get_mut(app_id)
      .and_then(|app_instances| app_instances.find_sticky(sticky_session))
      .map(|b| {
        let ref mut backend = *b.borrow_mut();
        let conn = backend.try_connect();
        info!("Connecting {} -> {:?} using session {}", app_id, (backend.address, backend.active_connections, backend.failures), sticky_session);
        if backend.failures >= max_failures_per_backend {
          error!("backend {:?} connections failed {} times, disabling it", (backend.address, backend.active_connections), backend.failures);
        }

        conn.map(|c| (b.clone(), c))
    });

    if let Some(res) = sticky_conn {
      return res;
    } else {
      debug!("Couldn't find a backend corresponding to sticky_session {} for app {}", sticky_session, app_id);
      return self.backend_from_app_id(app_id);
    }
  }
}

const max_failures_per_backend: usize = 10;

pub struct BackendList {
  pub instances: Vec<Rc<RefCell<Backend>>>,
  pub next_id:   u32,
}

impl BackendList {
  pub fn new() -> BackendList {
    BackendList {
      instances: Vec::new(),
      next_id:   0,
    }
  }

  pub fn add_instance(&mut self, instance_address: &SocketAddr) {
    if self.instances.iter().find(|b| &(*b.borrow()).address == instance_address).is_none() {
      let backend = Rc::new(RefCell::new(Backend::new(*instance_address, self.next_id)));
      self.instances.push(backend);
      self.next_id += 1;
    }
  }

  pub fn remove_instance(&mut self, instance_address: &SocketAddr) {
    self.instances.retain(|backend| &(*backend.borrow()).address != instance_address);
  }

  pub fn find_instance(&mut self, instance_address: &SocketAddr) -> Option<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut().find(|backend| &(*backend.borrow()).address == instance_address)
  }

  pub fn find_sticky(&mut self, sticky_session: u32) -> Option<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut().find(|b| {
      let backend = &*b.borrow();
      backend.id == sticky_session && backend.can_open(max_failures_per_backend)
    })
  }

  pub fn available_instances(&mut self) -> Vec<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut()
      .filter(|backend| (*backend.borrow()).can_open(max_failures_per_backend))
      .collect()
  }
}
<commit_msg>move the random load balancing to BackendList<commit_after>use std::rc::Rc;
use std::cell::RefCell;
use std::net::SocketAddr;
use std::collections::HashMap;
use rand::random;
use mio::net::TcpStream;

use network::{AppId,Backend,ConnectionError};

pub struct BackendMap {
  pub instances:    HashMap<AppId, BackendList>,
  pub max_failures: usize,
}

impl BackendMap {
  pub fn new() -> BackendMap {
    BackendMap {
      instances:    HashMap::new(),
      max_failures: 3,
    }
  }

  pub fn add_instance(&mut self, app_id: &str, instance_address: &SocketAddr) {
    self.instances.entry(app_id.to_string()).or_insert(BackendList::new()).add_instance(instance_address);
  }

  pub fn remove_instance(&mut self, app_id: &str, instance_address: &SocketAddr) {
    if let Some(instances) = self.instances.get_mut(app_id) {
      instances.remove_instance(instance_address);
    } else {
      error!("Instance was already removed");
    }
  }

  pub fn close_backend_connection(&mut self, app_id: &str, addr: &SocketAddr) {
    if let Some(app_instances) = self.instances.get_mut(app_id) {
      if let Some(ref mut backend) = app_instances.find_instance(addr) {
        (*backend.borrow_mut()).dec_connections();
      }
    }
  }

  pub fn backend_from_app_id(&mut self, app_id: &str) -> Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError> {
    if let Some(ref mut app_instances) = self.instances.get_mut(app_id) {
      if app_instances.instances.len() == 0 {
        return Err(ConnectionError::NoBackendAvailable);
      }

      for _ in 0..self.max_failures {
        if let Some(ref mut b) = app_instances.next_available_instance() {
          let ref mut backend = *b.borrow_mut();
          info!("Connecting {} -> {:?}", app_id, (backend.address, backend.active_connections, backend.failures));
          let conn = backend.try_connect();
          if backend.failures >= max_failures_per_backend {
            error!("backend {:?} connections failed {} times, disabling it", (backend.address, backend.active_connections), backend.failures);
          }

          return conn.map(|c| (b.clone(), c));
        } else {
          error!("no more available backends for app {}", app_id);
          return Err(ConnectionError::NoBackendAvailable);
        }
      }
      Err(ConnectionError::NoBackendAvailable)
    } else {
      Err(ConnectionError::NoBackendAvailable)
    }
  }

  pub fn backend_from_sticky_session(&mut self, app_id: &str, sticky_session: u32) -> Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError> {
    let sticky_conn: Option<Result<(Rc<RefCell<Backend>>,TcpStream),ConnectionError>> = self.instances
      .get_mut(app_id)
      .and_then(|app_instances| app_instances.find_sticky(sticky_session))
      .map(|b| {
        let ref mut backend = *b.borrow_mut();
        let conn = backend.try_connect();
        info!("Connecting {} -> {:?} using session {}", app_id, (backend.address, backend.active_connections, backend.failures), sticky_session);
        if backend.failures >= max_failures_per_backend {
          error!("backend {:?} connections failed {} times, disabling it", (backend.address, backend.active_connections), backend.failures);
        }

        conn.map(|c| (b.clone(), c))
    });

    if let Some(res) = sticky_conn {
      return res;
    } else {
      debug!("Couldn't find a backend corresponding to sticky_session {} for app {}", sticky_session, app_id);
      return self.backend_from_app_id(app_id);
    }
  }
}

const max_failures_per_backend: usize = 10;

pub struct BackendList {
  pub instances: Vec<Rc<RefCell<Backend>>>,
  pub next_id:   u32,
}

impl BackendList {
  pub fn new() -> BackendList {
    BackendList {
      instances: Vec::new(),
      next_id:   0,
    }
  }

  pub fn add_instance(&mut self, instance_address: &SocketAddr) {
    if self.instances.iter().find(|b| &(*b.borrow()).address == instance_address).is_none() {
      let backend = Rc::new(RefCell::new(Backend::new(*instance_address, self.next_id)));
      self.instances.push(backend);
      self.next_id += 1;
    }
  }

  pub fn remove_instance(&mut self, instance_address: &SocketAddr) {
    self.instances.retain(|backend| &(*backend.borrow()).address != instance_address);
  }

  pub fn find_instance(&mut self, instance_address: &SocketAddr) -> Option<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut().find(|backend| &(*backend.borrow()).address == instance_address)
  }

  pub fn find_sticky(&mut self, sticky_session: u32) -> Option<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut().find(|b| {
      let backend = &*b.borrow();
      backend.id == sticky_session && backend.can_open(max_failures_per_backend)
    })
  }

  pub fn available_instances(&mut self) -> Vec<&mut Rc<RefCell<Backend>>> {
    self.instances.iter_mut()
      .filter(|backend| (*backend.borrow()).can_open(max_failures_per_backend))
      .collect()
  }

  pub fn next_available_instance(&mut self) -> Option<&mut Rc<RefCell<Backend>>> {
    let mut instances:Vec<&mut Rc<RefCell<Backend>>> = self.available_instances();
    if instances.is_empty() {
      return None;
    }

    let rnd = random::<usize>();
    let idx = rnd % instances.len();

    Some(instances.remove(idx))
  }
}
<|endoftext|>


<commit_before>//! Code to adapt between the outgoing `std::io` and the incoming `std::rt::io` traits.
//! This code will be rendered obsolete before Rust 1.0.

use std::comm::{Port, Chan, SharedChan};
use std::{rt, task, vec};
use extra::{net, uv_iotask};

static READ_SIZE : uint = 1024;

/// Adapter of extra::net stuff to std::rt::io::net stuff
pub struct ExtraNetTcpListener {
    accept_ch: Chan<()>,
    stream_po: Port<Option<ExtraNetTcpStream>>,
}

enum StreamOperationRequest { Read(uint), Write }

impl ExtraNetTcpListener {
    pub fn bind(addr: rt::io::net::ip::IpAddr) -> Option<ExtraNetTcpListener> {
        match addr {
            rt::io::net::ip::Ipv4(a, b, c, d, p) => {
                let (accept_po, accept_ch) = stream();
                let (stream_po, stream_ch) = stream();
                let stream_ch = SharedChan::new(stream_ch);
                do spawn {
                    net::tcp::listen(
                        net::ip::v4::parse_addr(fmt!("%?.%?.%?.%?", a, b, c, d)),
                        p as uint,
                        100,
                        &uv_iotask::spawn_iotask(task::task()),
                        |_kill_ch| { },
                        |new_conn, _kill_ch| {
                            // An accept request has been sent, try accepting it
                            accept_po.recv();
                            let child_stream_ch = stream_ch.clone();
                            do task::spawn_supervised {
                                let accept_result = net::tcp::accept(new_conn);
                                match accept_result {
                                    Ok(socket) => {
                                        let (req_po, req_ch) = stream();
                                        let (write_po, write_ch) = stream();
                                        let (read_po, read_ch) = stream();
                                        let sockbuf = net::tcp::socket_buf(socket);
                                        let stream = ExtraNetTcpStream {
                                            req_ch: req_ch,
                                            write_ch: write_ch,
                                            read_po: read_po,
                                        };
                                        child_stream_ch.send(Some(stream));
                                        loop {
                                            match req_po.try_recv() {
                                                Some(Read(1)) => {
                                                    match sockbuf.read_byte() {
                                                        v if v >= 0 && v <= 255 => {
                                                            read_ch.send((~[v as u8], 1u));
                                                        },
                                                        _ => {
                                                            read_ch.send((~[0u8], 0u));
                                                        }
                                                    }
                                                },
                                                Some(Read(len)) => {
                                                    //let mut buf = vec::with_capacity(len);
                                                    let mut buf = vec::from_elem(len, 0);
                                                    let size = sockbuf.read(buf, len);
                                                    //let mut buf = ~[0u8, ..READ_SIZE];
                                                    //let size = sockbuf.read(buf, READ_SIZE);
                                                    read_ch.send((buf, size))
                                                },
                                                Some(Write) => {
                                                    sockbuf.write(write_po.recv());
                                                }
                                                None => {
                                                    // Closed: request completed
                                                    break;
                                                }
                                            }
                                        }
                                    }
                                    Err(err) => {
                                        debug!("Error accepting, %?", err);
                                        child_stream_ch.send(None);
                                    }
                                }
                            }
                        }
                    );
                }

                Some(ExtraNetTcpListener {
                    accept_ch: accept_ch,
                    stream_po: stream_po,
                })
            },
            rt::io::net::ip::Ipv6(_, _, _, _, _, _, _, _, _port) => fail!("no IPv6 support")
        }
    }
}

impl rt::io::Listener<ExtraNetTcpStream> for ExtraNetTcpListener {
    pub fn accept(&mut self) -> Option<ExtraNetTcpStream> {
        self.accept_ch.send(());

        match self.stream_po.try_recv() {
            Some(optstream) => optstream,
            None => None,
        }
    }
}

pub struct ExtraNetTcpStream {
    req_ch: Chan<StreamOperationRequest>,
    write_ch: Chan<~[u8]>,
    read_po: Port<(~[u8], uint)>,
}

impl rt::io::Reader for ExtraNetTcpStream {
    //io::Reader has fn read(&self, buf: &mut [u8], len: uint) -> uint
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.req_ch.send(Read(buf.len()));
        let (read_buf, num_bytes_read) = self.read_po.recv();
        let mut i = 0;
        while i < num_bytes_read {
            buf[i] = read_buf[i];
            i += 1;
        }
        return Some(num_bytes_read);
    }

    fn eof(&mut self) -> bool { fail!("ExtraNetTcpStream.eof() is not supported") }
}

impl rt::io::Writer for ExtraNetTcpStream {
    fn write(&mut self, buf: &[u8]) {
        self.req_ch.send(Write);
        self.write_ch.send(buf.to_owned())
    }

    fn flush(&mut self) { fail!("ExtraNetTcpStream.flush() is not supported") }
}
<commit_msg>Work around a new integer type inference bug.<commit_after>//! Code to adapt between the outgoing `std::io` and the incoming `std::rt::io` traits.
//! This code will be rendered obsolete before Rust 1.0.

use std::comm::{Port, Chan, SharedChan};
use std::{rt, task, vec};
use extra::{net, uv_iotask};

static READ_SIZE : uint = 1024;

/// Adapter of extra::net stuff to std::rt::io::net stuff
pub struct ExtraNetTcpListener {
    accept_ch: Chan<()>,
    stream_po: Port<Option<ExtraNetTcpStream>>,
}

enum StreamOperationRequest { Read(uint), Write }

impl ExtraNetTcpListener {
    pub fn bind(addr: rt::io::net::ip::IpAddr) -> Option<ExtraNetTcpListener> {
        match addr {
            rt::io::net::ip::Ipv4(a, b, c, d, p) => {
                let (accept_po, accept_ch) = stream();
                let (stream_po, stream_ch) = stream();
                let stream_ch = SharedChan::new(stream_ch);
                do spawn {
                    net::tcp::listen(
                        net::ip::v4::parse_addr(fmt!("%?.%?.%?.%?", a, b, c, d)),
                        p as uint,
                        100,
                        &uv_iotask::spawn_iotask(task::task()),
                        |_kill_ch| { },
                        |new_conn, _kill_ch| {
                            // An accept request has been sent, try accepting it
                            accept_po.recv();
                            let child_stream_ch = stream_ch.clone();
                            do task::spawn_supervised {
                                let accept_result = net::tcp::accept(new_conn);
                                match accept_result {
                                    Ok(socket) => {
                                        let (req_po, req_ch) = stream();
                                        let (write_po, write_ch) = stream();
                                        let (read_po, read_ch) = stream();
                                        let sockbuf = net::tcp::socket_buf(socket);
                                        let stream = ExtraNetTcpStream {
                                            req_ch: req_ch,
                                            write_ch: write_ch,
                                            read_po: read_po,
                                        };
                                        child_stream_ch.send(Some(stream));
                                        loop {
                                            match req_po.try_recv() {
                                                Some(Read(1)) => {
                                                    match sockbuf.read_byte() {
                                                        v if v >= 0 && v <= 255 => {
                                                            read_ch.send((~[v as u8], 1u));
                                                        },
                                                        _ => {
                                                            read_ch.send((~[0u8], 0u));
                                                        }
                                                    }
                                                },
                                                Some(Read(len)) => {
                                                    //let mut buf = vec::with_capacity(len);
                                                    let mut buf = vec::from_elem(len, 0u8);
                                                    let size = sockbuf.read(buf, len);
                                                    //let mut buf = ~[0u8, ..READ_SIZE];
                                                    //let size = sockbuf.read(buf, READ_SIZE);
                                                    read_ch.send((buf, size))
                                                },
                                                Some(Write) => {
                                                    sockbuf.write(write_po.recv());
                                                }
                                                None => {
                                                    // Closed: request completed
                                                    break;
                                                }
                                            }
                                        }
                                    }
                                    Err(err) => {
                                        debug!("Error accepting, %?", err);
                                        child_stream_ch.send(None);
                                    }
                                }
                            }
                        }
                    );
                }

                Some(ExtraNetTcpListener {
                    accept_ch: accept_ch,
                    stream_po: stream_po,
                })
            },
            rt::io::net::ip::Ipv6(_, _, _, _, _, _, _, _, _port) => fail!("no IPv6 support")
        }
    }
}

impl rt::io::Listener<ExtraNetTcpStream> for ExtraNetTcpListener {
    pub fn accept(&mut self) -> Option<ExtraNetTcpStream> {
        self.accept_ch.send(());

        match self.stream_po.try_recv() {
            Some(optstream) => optstream,
            None => None,
        }
    }
}

pub struct ExtraNetTcpStream {
    req_ch: Chan<StreamOperationRequest>,
    write_ch: Chan<~[u8]>,
    read_po: Port<(~[u8], uint)>,
}

impl rt::io::Reader for ExtraNetTcpStream {
    //io::Reader has fn read(&self, buf: &mut [u8], len: uint) -> uint
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.req_ch.send(Read(buf.len()));
        let (read_buf, num_bytes_read) = self.read_po.recv();
        let mut i = 0;
        while i < num_bytes_read {
            buf[i] = read_buf[i];
            i += 1;
        }
        return Some(num_bytes_read);
    }

    fn eof(&mut self) -> bool { fail!("ExtraNetTcpStream.eof() is not supported") }
}

impl rt::io::Writer for ExtraNetTcpStream {
    fn write(&mut self, buf: &[u8]) {
        self.req_ch.send(Write);
        self.write_ch.send(buf.to_owned())
    }

    fn flush(&mut self) { fail!("ExtraNetTcpStream.flush() is not supported") }
}
<|endoftext|>


<commit_before>use super::{Open, Sink, SinkAsBytes};
use crate::config::AudioFormat;
use crate::convert::Converter;
use crate::decoder::AudioPacket;
use std::fs::OpenOptions;
use std::io::{self, Write};

pub struct StdoutSink {
    output: Option<Box<dyn Write>>,
    path: Option<String>,
    format: AudioFormat,
}

impl Open for StdoutSink {
    fn open(path: Option<String>, format: AudioFormat) -> Self {
        info!("Using pipe sink with format: {:?}", format);
        Self {
            output: None,
            path: path,
            format,
        }
    }
}

impl Sink for StdoutSink {
    fn start(&mut self) -> io::Result<()> {
        if self.output.is_none() {
            let output: Box<dyn Write> = match self.path.as_deref() {
                Some(path) => {
                    let open_op = OpenOptions::new()
                        .write(true)
                        .open(path)
                        .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
                    Box::new(open_op)
                }
                None => Box::new(io::stdout()),
            };

            self.output = Some(output);
        }

        Ok(())
    }

    sink_as_bytes!();
}

impl SinkAsBytes for StdoutSink {
    fn write_bytes(&mut self, data: &[u8]) -> io::Result<()> {
        match self.output.as_deref_mut() {
            Some(output) => {
                output.write_all(data)?;
                output.flush()?;
            }
            None => {
                return Err(io::Error::new(io::ErrorKind::Other, "Output is None"));
            }
        }

        Ok(())
    }
}

impl StdoutSink {
    pub const NAME: &'static str = "pipe";
}
<commit_msg>Fix clippy warning<commit_after>use super::{Open, Sink, SinkAsBytes};
use crate::config::AudioFormat;
use crate::convert::Converter;
use crate::decoder::AudioPacket;
use std::fs::OpenOptions;
use std::io::{self, Write};

pub struct StdoutSink {
    output: Option<Box<dyn Write>>,
    path: Option<String>,
    format: AudioFormat,
}

impl Open for StdoutSink {
    fn open(path: Option<String>, format: AudioFormat) -> Self {
        info!("Using pipe sink with format: {:?}", format);
        Self {
            output: None,
            path,
            format,
        }
    }
}

impl Sink for StdoutSink {
    fn start(&mut self) -> io::Result<()> {
        if self.output.is_none() {
            let output: Box<dyn Write> = match self.path.as_deref() {
                Some(path) => {
                    let open_op = OpenOptions::new()
                        .write(true)
                        .open(path)
                        .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
                    Box::new(open_op)
                }
                None => Box::new(io::stdout()),
            };

            self.output = Some(output);
        }

        Ok(())
    }

    sink_as_bytes!();
}

impl SinkAsBytes for StdoutSink {
    fn write_bytes(&mut self, data: &[u8]) -> io::Result<()> {
        match self.output.as_deref_mut() {
            Some(output) => {
                output.write_all(data)?;
                output.flush()?;
            }
            None => {
                return Err(io::Error::new(io::ErrorKind::Other, "Output is None"));
            }
        }

        Ok(())
    }
}

impl StdoutSink {
    pub const NAME: &'static str = "pipe";
}
<|endoftext|>


<commit_before>#[cfg(feature="pkg-config")]
extern crate pkg_config;

fn main() {
    if !build_pkgconfig() {
        if cfg!(feature="use_mac_framework") {
            println!("cargo:rustc-flags=-l framework=SDL2");
        } else {
            println!("cargo:rustc-flags=-l SDL2");
        }
    }
}

#[cfg(not(feature="pkg-config"))]
fn build_pkgconfig() -> bool {
    false
}

#[cfg(feature="pkg-config")]
fn build_pkgconfig() -> bool {
    if pkg_config::find_library("sdl2").is_err() {
        panic!("Could not find SDL2 via pkgconfig");
    }
    true
}
<commit_msg>Only attempt to link against a framework if building for mac<commit_after>#[cfg(feature="pkg-config")]
extern crate pkg_config;

fn main() {
    if !build_pkgconfig() {
        let target = ::std::env::var("TARGET").expect("Cargo build scripts always have TARGET");
        let target_os = target.splitn(3, "-").nth(2).unwrap();

        if cfg!(feature="use_mac_framework") && target_os == "darwin" {
            println!("cargo:rustc-flags=-l framework=SDL2");
        } else {
            println!("cargo:rustc-flags=-l SDL2");
        }
    }
}

#[cfg(not(feature="pkg-config"))]
fn build_pkgconfig() -> bool {
    false
}

#[cfg(feature="pkg-config")]
fn build_pkgconfig() -> bool {
    if pkg_config::find_library("sdl2").is_err() {
        panic!("Could not find SDL2 via pkgconfig");
    }
    true
}
<|endoftext|>