<commit_before>use std::{cmp::Ordering, fmt, path::Path};

use async_trait::async_trait;
use serde::{Deserialize, Serialize};

use crate::{build::BuildStep, bundle::KbdgenBundle};

const REPOSITORY: &str = "repo";
const HOSTING_APP: &str = "HostingApp";
const INFO_PLIST_STRINGS: &str = "InfoPlist.strings";

const KEYBOARD: &str = "Keyboard";
const INFO_PLIST: &str = "Info.plist";

const SETTINGS_BUNDLE: &str = "Settings.bundle";
const ROOT_PLIST: &str = "Root.plist";
const ENTITLEMENTS_EXTENSION: &str = ".entitlements";

#[derive(Serialize, Deserialize, Debug)]
pub struct XcodeInfoPlist {
    #[serde(rename = "CFBundleName")]
    cf_bundle_name: String,
    #[serde(rename = "CFBundleDisplayName")]
    cf_bundle_display_name: String,
}

impl fmt::Display for XcodeInfoPlist {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.write_str(&format!(
            "\"CFBundleName\" = {:?};\n\"CFBundleDisplayName\" = {:?};",
            self.cf_bundle_name, self.cf_bundle_display_name
        ))
    }
}

// LAYOUT PLIST START

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlistExtensionAttributes {
    #[serde(rename = "IsASCIICapable")]
    is_ascii_capable: bool,
    #[serde(rename = "PrefersRightToLeft")]
    prefers_right_to_left: bool,
    #[serde(rename = "PrimaryLanguage")]
    primary_language: String,
    #[serde(rename = "RequestsOpenAccess")]
    requests_open_access: bool,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlistExtension {
    #[serde(rename = "NSExtensionAttributes")]
    ns_extension_attributes: LayoutInfoPlistExtensionAttributes,
    #[serde(rename = "NSExtensionPointIdentifier")]
    ns_extension_point_identifier: String,
    #[serde(rename = "NSExtensionPrincipalClass")]
    ns_extension_principal_class: String,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlist {
    #[serde(rename = "DivvunSpellerPath")]
    divvun_speller_path: String,
    #[serde(rename = "DivvunSpellerPackageKey")]
    divvun_speller_package_key: String,
    #[serde(rename = "CFBundleDevelopmentRegion")]
    cf_bundle_development_region: String,
    #[serde(rename = "CFBundleDisplayName")]
    cf_bundle_display_name: String,
    #[serde(rename = "CFBundleExecutable")]
    cf_bundle_executable: String,
    #[serde(rename = "CFBundleIdentifier")]
    cf_bundle_identifier: String,
    #[serde(rename = "CFBundleInfoDictionaryVersion")]
    cf_bundle_info_dictionary_version: String,
    #[serde(rename = "CFBundleName")]
    cf_bundle_name: String,
    #[serde(rename = "CFBundlePackageType")]
    cf_bundle_package_type: String,
    #[serde(rename = "CFBundleShortVersionString")]
    cf_bundle_short_version_string: String,
    #[serde(rename = "CFBundleSignature")]
    cf_bundle_signature: String,
    #[serde(rename = "CFBundleVersion")]
    cf_bundle_version: String,
    #[serde(rename = "DivvunKeyboardIndex")]
    divvun_keyboard_index: usize,
    #[serde(rename = "ITSAppUsesNonExemptEncryption")]
    its_app_uses_non_exempt_encryption: bool,
    #[serde(rename = "LSApplicationQueriesSchemes")]
    ls_application_queries_schemes: Vec<String>,
    #[serde(rename = "NSExtension")]
    ns_extension: LayoutInfoPlistExtension,
}

// LAYOUT PLIST END

#[derive(Serialize, Deserialize, Debug)]
pub struct EntitlementsDict {
    #[serde(rename = "com.apple.security.application-groups")]
    com_apple_security_application_groups: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct PreferenceSpecifier {
    #[serde(rename = "Type")]
    preference_type: String,
    #[serde(rename = "Title")]
    title: String,
    #[serde(rename = "Key")]
    key: String,
    #[serde(rename = "DefaultValue")]
    default_value: bool,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct SettingsRootDict {
    #[serde(rename = "StringsTable")]
    strings_table: String,
    #[serde(rename = "PreferenceSpecifiers")]
    preference_specifiers: Vec<PreferenceSpecifier>,
    #[serde(rename = "ApplicationGroupContainerIdentifier")]
    application_group_container_identifier: String,
}

pub fn replace_all_occurances(input: String, character: char, replace_with: char) -> String {
    input
        .as_str()
        .chars()
        .map(|curr| {
            if curr.cmp(&character) != Ordering::Equal {
                curr
            } else {
                replace_with
            }
        })
        .into_iter()
        .collect::<String>()
}

pub struct GenerateXcode;

#[async_trait(?Send)]
impl BuildStep for GenerateXcode {
    async fn build(&self, bundle: &KbdgenBundle, output_path: &Path) {
        let repository_path = output_path.join(REPOSITORY);
        let hosting_app_path = repository_path.join(HOSTING_APP);
        let keyboard_path = repository_path.join(KEYBOARD);

        for (layout_index, (language_tag, layout)) in bundle.layouts.iter().enumerate() {
            if let Some(target) = &bundle.targets.ios {
                if let Some(_ios_layout) = &layout.i_os {
                    // GENERATE LOCALES
                    for (locale_name, locale_info) in &bundle.project.locales {
                        let locale_name = if locale_name == "en" {
                            "Base"
                        } else {
                            locale_name
                        };
                        let locale_path = hosting_app_path.join(&format!("{}.lproj", locale_name));
                        let info_path = locale_path.join(INFO_PLIST_STRINGS);
                        println!("{:?}", locale_path);

                        std::fs::create_dir_all(&locale_path).unwrap();

                        let info_plist = XcodeInfoPlist {
                            cf_bundle_name: locale_info.name.to_string(),
                            cf_bundle_display_name: locale_info.name.to_string(),
                        };
                        std::fs::write(info_path, info_plist.to_string()).unwrap();
                    }

                    // GENERATE LAYOUTS

                    let keyboard_name = replace_all_occurances(
                        bundle
                            .project
                            .locales
                            .get("en")
                            .unwrap()
                            .name
                            .to_lowercase(),
                        ' ',
                        '-',
                    );

                    let info_plist_template = keyboard_path.join(INFO_PLIST);

                    let layout_path = keyboard_path.join(keyboard_name);
                    let layout_info_plist_path = layout_path.join(INFO_PLIST);

                    std::fs::create_dir_all(&layout_path).unwrap();

                    let mut parsed_plist: LayoutInfoPlist =
                        plist::from_file(info_plist_template.clone()).expect("valid stuff");

                    parsed_plist.cf_bundle_display_name = layout.autonym().to_string();
                    parsed_plist.cf_bundle_short_version_string = target.version.clone();
                    parsed_plist.cf_bundle_version = target.build.clone();
                    parsed_plist.ls_application_queries_schemes[0] = target.package_id.clone();
                    parsed_plist
                        .ns_extension
                        .ns_extension_attributes
                        .primary_language = language_tag.to_string();
                    parsed_plist.divvun_keyboard_index = layout_index;

                    plist::to_file_xml(layout_info_plist_path.clone(), &parsed_plist).unwrap();

                    // UPDATE ENTITLEMENTS
                    let new_entitlements = format!("{}.{}", "group", target.package_id);

                    // KEYBOARD
                    let keyboard_entitlements_path =
                        keyboard_path.join(format!("{}{}", KEYBOARD, ENTITLEMENTS_EXTENSION));
                    let mut keyboard_entitlements: EntitlementsDict =
                        plist::from_file(keyboard_entitlements_path.clone()).expect("valid stuff");
                    keyboard_entitlements.com_apple_security_application_groups =
                        vec![new_entitlements.clone()];
                    plist::to_file_xml(keyboard_entitlements_path.clone(), &keyboard_entitlements)
                        .unwrap();

                    // HOSTING APP
                    let hosting_app_entitlements_path =
                        hosting_app_path.join(format!("{}{}", HOSTING_APP, ENTITLEMENTS_EXTENSION));
                    let mut hosting_app_entitlements: EntitlementsDict =
                        plist::from_file(hosting_app_entitlements_path.clone())
                            .expect("valid stuff");
                    hosting_app_entitlements.com_apple_security_application_groups =
                        vec![new_entitlements.clone()];
                    plist::to_file_xml(
                        hosting_app_entitlements_path.clone(),
                        &hosting_app_entitlements,
                    )
                    .unwrap();

                    // ROOT PLIST
                    let root_plist_path = hosting_app_path.join(SETTINGS_BUNDLE).join(ROOT_PLIST);
                    let mut root_plist: SettingsRootDict =
                        plist::from_file(root_plist_path.clone()).expect("valid stuff");
                    root_plist.application_group_container_identifier = new_entitlements.clone();
                    plist::to_file_xml(root_plist_path.clone(), &root_plist).unwrap();
                }
            }
        }
    }
}
<commit_msg>entitlements and hosting app plist<commit_after>use std::{cmp::Ordering, fmt, path::Path};

use async_trait::async_trait;
use serde::{Deserialize, Serialize};

use crate::{build::BuildStep, bundle::KbdgenBundle};

const REPOSITORY: &str = "repo";
const HOSTING_APP: &str = "HostingApp";
const INFO_PLIST_STRINGS: &str = "InfoPlist.strings";

const KEYBOARD: &str = "Keyboard";
const INFO_PLIST: &str = "Info.plist";

const SETTINGS_BUNDLE: &str = "Settings.bundle";
const ROOT_PLIST: &str = "Root.plist";
const ENTITLEMENTS_EXTENSION: &str = ".entitlements";

#[derive(Serialize, Deserialize, Debug)]
pub struct XcodeInfoPlist {
    #[serde(rename = "CFBundleName")]
    cf_bundle_name: String,
    #[serde(rename = "CFBundleDisplayName")]
    cf_bundle_display_name: String,
}

impl fmt::Display for XcodeInfoPlist {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.write_str(&format!(
            "\"CFBundleName\" = {:?};\n\"CFBundleDisplayName\" = {:?};",
            self.cf_bundle_name, self.cf_bundle_display_name
        ))
    }
}

// LAYOUT PLIST START

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlistExtensionAttributes {
    #[serde(rename = "IsASCIICapable")]
    is_ascii_capable: bool,
    #[serde(rename = "PrefersRightToLeft")]
    prefers_right_to_left: bool,
    #[serde(rename = "PrimaryLanguage")]
    primary_language: String,
    #[serde(rename = "RequestsOpenAccess")]
    requests_open_access: bool,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlistExtension {
    #[serde(rename = "NSExtensionAttributes")]
    ns_extension_attributes: LayoutInfoPlistExtensionAttributes,
    #[serde(rename = "NSExtensionPointIdentifier")]
    ns_extension_point_identifier: String,
    #[serde(rename = "NSExtensionPrincipalClass")]
    ns_extension_principal_class: String,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct LayoutInfoPlist {
    #[serde(rename = "DivvunSpellerPath")]
    divvun_speller_path: String,
    #[serde(rename = "DivvunSpellerPackageKey")]
    divvun_speller_package_key: String,
    #[serde(rename = "CFBundleDevelopmentRegion")]
    cf_bundle_development_region: String,
    #[serde(rename = "CFBundleDisplayName")]
    cf_bundle_display_name: String,
    #[serde(rename = "CFBundleExecutable")]
    cf_bundle_executable: String,
    #[serde(rename = "CFBundleIdentifier")]
    cf_bundle_identifier: String,
    #[serde(rename = "CFBundleInfoDictionaryVersion")]
    cf_bundle_info_dictionary_version: String,
    #[serde(rename = "CFBundleName")]
    cf_bundle_name: String,
    #[serde(rename = "CFBundlePackageType")]
    cf_bundle_package_type: String,
    #[serde(rename = "CFBundleShortVersionString")]
    cf_bundle_short_version_string: String,
    #[serde(rename = "CFBundleSignature")]
    cf_bundle_signature: String,
    #[serde(rename = "CFBundleVersion")]
    cf_bundle_version: String,
    #[serde(rename = "DivvunKeyboardIndex")]
    divvun_keyboard_index: usize,
    #[serde(rename = "ITSAppUsesNonExemptEncryption")]
    its_app_uses_non_exempt_encryption: bool,
    #[serde(rename = "LSApplicationQueriesSchemes")]
    ls_application_queries_schemes: Vec<String>,
    #[serde(rename = "NSExtension")]
    ns_extension: LayoutInfoPlistExtension,
}

// LAYOUT PLIST END

#[derive(Serialize, Deserialize, Debug)]
pub struct EntitlementsDict {
    #[serde(rename = "com.apple.security.application-groups")]
    com_apple_security_application_groups: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct PreferenceSpecifier {
    #[serde(rename = "Type")]
    preference_type: String,
    #[serde(rename = "Title")]
    title: String,
    #[serde(rename = "Key")]
    key: String,
    #[serde(rename = "DefaultValue")]
    default_value: bool,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct SettingsRootDict {
    #[serde(rename = "StringsTable")]
    strings_table: String,
    #[serde(rename = "PreferenceSpecifiers")]
    preference_specifiers: Vec<PreferenceSpecifier>,
    #[serde(rename = "ApplicationGroupContainerIdentifier")]
    application_group_container_identifier: String,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct BundleSchemes {
    CFBundleURLSchemes: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct HostingPlist {
    CFBundleDevelopmentRegion: String,
    CFBundleDisplayName: String,
    CFBundleExecutable: String,
    CFBundleIdentifier: String,
    CFBundleInfoDictionaryVersion: String,
    CFBundleName: String,
    CFBundlePackageType: String,
    CFBundleShortVersionString: String,
    CFBundleSignature: String,
    CFBundleURLTypes: Vec<BundleSchemes>,
    CFBundleVersion: String,
    ITSAppUsesNonExemptEncryption: bool,
    LSApplicationQueriesSchemes: Vec<String>,
    LSRequiresIPhoneOS: bool,
    UIBackgroundModes: Vec<String>,
    UILaunchStoryboardName: String,
    UIRequiredDeviceCapabilities: Vec<String>,
    UISupportedInterfaceOrientations: Vec<String>,
    UIUserInterfaceStyle: String,
}

pub fn replace_all_occurances(input: String, character: char, replace_with: char) -> String {
    input
        .as_str()
        .chars()
        .map(|curr| {
            if curr.cmp(&character) != Ordering::Equal {
                curr
            } else {
                replace_with
            }
        })
        .into_iter()
        .collect::<String>()
}

pub struct GenerateXcode;

#[async_trait(?Send)]
impl BuildStep for GenerateXcode {
    async fn build(&self, bundle: &KbdgenBundle, output_path: &Path) {
        let repository_path = output_path.join(REPOSITORY);
        let hosting_app_path = repository_path.join(HOSTING_APP);
        let keyboard_path = repository_path.join(KEYBOARD);

        for (layout_index, (language_tag, layout)) in bundle.layouts.iter().enumerate() {
            if let Some(target) = &bundle.targets.ios {
                if let Some(_ios_layout) = &layout.i_os {
                    // GENERATE LOCALES
                    for (locale_name, locale_info) in &bundle.project.locales {
                        let locale_name = if locale_name == "en" {
                            "Base"
                        } else {
                            locale_name
                        };
                        let locale_path = hosting_app_path.join(&format!("{}.lproj", locale_name));
                        let info_path = locale_path.join(INFO_PLIST_STRINGS);
                        println!("{:?}", locale_path);

                        std::fs::create_dir_all(&locale_path).unwrap();

                        let info_plist = XcodeInfoPlist {
                            cf_bundle_name: locale_info.name.to_string(),
                            cf_bundle_display_name: locale_info.name.to_string(),
                        };
                        std::fs::write(info_path, info_plist.to_string()).unwrap();
                    }

                    // GENERATE LAYOUTS

                    // KEYBOARD PLIST
                    let keyboard_folder_name = replace_all_occurances(
                        bundle
                            .project
                            .locales
                            .get("en")
                            .unwrap()
                            .name
                            .to_lowercase(),
                        ' ',
                        '-',
                    );

                    let info_plist_template = keyboard_path.join(INFO_PLIST);

                    let layout_path = keyboard_path.join(keyboard_folder_name);
                    let layout_info_plist_path = layout_path.join(INFO_PLIST);

                    std::fs::create_dir_all(&layout_path).unwrap();

                    let mut parsed_keyboard_plist: LayoutInfoPlist =
                        plist::from_file(info_plist_template.clone()).expect("valid stuff");

                    parsed_keyboard_plist.cf_bundle_display_name = layout.autonym().to_string();
                    parsed_keyboard_plist.cf_bundle_short_version_string = target.version.clone();
                    parsed_keyboard_plist.cf_bundle_version = target.build.clone();
                    parsed_keyboard_plist.ls_application_queries_schemes[0] =
                        target.package_id.clone();
                    parsed_keyboard_plist
                        .ns_extension
                        .ns_extension_attributes
                        .primary_language = language_tag.to_string();
                    parsed_keyboard_plist.divvun_keyboard_index = layout_index;

                    plist::to_file_xml(layout_info_plist_path.clone(), &parsed_keyboard_plist)
                        .unwrap();

                    // HOSTING APP PLIST
                    let hosting_app_plist_path = hosting_app_path.join(INFO_PLIST);
                    let mut hosting_app_plist: HostingPlist =
                        plist::from_file(hosting_app_plist_path.clone()).expect("valid stuff");

                    hosting_app_plist.CFBundleDisplayName =
                        bundle.project.locales.get("en").unwrap().name.clone();
                    hosting_app_plist.CFBundleShortVersionString = target.version.clone();
                    hosting_app_plist.CFBundleVersion = target.build.clone();
                    hosting_app_plist.CFBundleURLTypes[0].CFBundleURLSchemes[0] =
                        target.package_id.clone();
                    hosting_app_plist.LSApplicationQueriesSchemes[0] = target.package_id.clone();

                    plist::to_file_xml(hosting_app_plist_path.clone(), &hosting_app_plist).unwrap();

                    // UPDATE ENTITLEMENTS
                    let new_entitlements = format!("{}.{}", "group", target.package_id);

                    // KEYBOARD
                    let keyboard_entitlements_path =
                        keyboard_path.join(format!("{}{}", KEYBOARD, ENTITLEMENTS_EXTENSION));
                    let mut keyboard_entitlements: EntitlementsDict =
                        plist::from_file(keyboard_entitlements_path.clone()).expect("valid stuff");
                    keyboard_entitlements.com_apple_security_application_groups =
                        vec![new_entitlements.clone()];
                    plist::to_file_xml(keyboard_entitlements_path.clone(), &keyboard_entitlements)
                        .unwrap();

                    // HOSTING APP
                    let hosting_app_entitlements_path =
                        hosting_app_path.join(format!("{}{}", HOSTING_APP, ENTITLEMENTS_EXTENSION));
                    let mut hosting_app_entitlements: EntitlementsDict =
                        plist::from_file(hosting_app_entitlements_path.clone())
                            .expect("valid stuff");
                    hosting_app_entitlements.com_apple_security_application_groups =
                        vec![new_entitlements.clone()];
                    plist::to_file_xml(
                        hosting_app_entitlements_path.clone(),
                        &hosting_app_entitlements,
                    )
                    .unwrap();

                    // ROOT PLIST
                    let root_plist_path = hosting_app_path.join(SETTINGS_BUNDLE).join(ROOT_PLIST);
                    let mut root_plist: SettingsRootDict =
                        plist::from_file(root_plist_path.clone()).expect("valid stuff");
                    root_plist.application_group_container_identifier = new_entitlements.clone();
                    plist::to_file_xml(root_plist_path.clone(), &root_plist).unwrap();
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use data::*;
use secstr::SecStr;
use rusterpassword::*;
use rustc_serialize::base64::{ToBase64, STANDARD};
use sodiumoxide::crypto::sign::ed25519;
use byteorder::{BigEndian, WriteBytesExt};
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use unix_socket::UnixStream;
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::env;
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::io::{Read, Write};
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::net::Shutdown;

pub enum Output {
    PrivateText(SecStr),
    OpenText(String),
    PrivateBinary(SecStr),
    Ed25519Keypair(Ed25519Usage, ed25519::PublicKey, ed25519::SecretKey),
}

fn pick_tpl(tpl: &PasswordTemplate) -> &'static [&'static str] {
    match *tpl {
       PasswordTemplate::Maximum  => TEMPLATES_MAXIMUM,
       PasswordTemplate::Long     => TEMPLATES_LONG,
       PasswordTemplate::Medium   => TEMPLATES_MEDIUM,
       PasswordTemplate::Short    => TEMPLATES_SHORT,
       PasswordTemplate::Basic    => TEMPLATES_BASIC,
       PasswordTemplate::Pin      => TEMPLATES_PIN,
    }
}

pub fn process_output(entry_name: &str, master_key: &SecStr, field: &Field) -> Result<Output> {
    match *field {
        Field::Derived { counter, ref site_name, ref usage } => {
            let site_seed = try!(gen_site_seed(master_key, &site_name.clone().unwrap_or(entry_name.to_string()), counter)
                                 .map_err(|_| Error::SeedGenerationError));
            match *usage {
                DerivedUsage::Password(ref tpl) =>
                    Ok(Output::PrivateText(gen_site_password(&site_seed, pick_tpl(tpl)))),
                DerivedUsage::Ed25519Key(ref keyusage) => {
                    let edseed = try!(ed25519::Seed::from_slice(site_seed.unsecure()).ok_or(Error::WrongDerivedKeyLength));
                    let (pubkey, seckey) = ed25519::keypair_from_seed(&edseed);
                    Ok(Output::Ed25519Keypair(*keyusage, pubkey, seckey))
                },
                DerivedUsage::RawKey =>
                    Ok(Output::PrivateBinary(site_seed)),
            }
        },
        Field::Stored { ref data, ref usage } =>
            match *usage {
                StoredUsage::Password =>
                    Ok(Output::PrivateText(SecStr::new(data.unsecure().to_vec()))),
                StoredUsage::Text =>
                    Ok(Output::OpenText(try!(String::from_utf8(data.unsecure().to_vec()))))
            }
    }
}

pub fn ssh_public_key_output(keypair: &Output, comment: &str) -> Result<String> {
    if let &Output::Ed25519Keypair(Ed25519Usage::SSH, ed25519::PublicKey(pubkey_bytes), _) = keypair {
        let mut raw = vec![];
        try!(raw.write_u32::<BigEndian>(11));
        raw.extend(b"ssh-ed25519");
        try!(raw.write_u32::<BigEndian>(ed25519::PUBLICKEYBYTES as u32));
        raw.extend(&pubkey_bytes);
        Ok("ssh-ed25519 ".to_string() + &raw.to_base64(STANDARD) + " " + comment)
    } else { Err(Error::InappropriateFormat) }
}

pub fn ssh_private_key_agent_message(keypair: &Output, comment: &str) -> Result<SecStr> {
    if let &Output::Ed25519Keypair(Ed25519Usage::SSH, ed25519::PublicKey(pubkey_bytes), ed25519::SecretKey(seckey_bytes)) = keypair {
        let mut msg = vec![17u8];
        try!(msg.write_u32::<BigEndian>(11));
        msg.extend(b"ssh-ed25519");
        try!(msg.write_u32::<BigEndian>(ed25519::PUBLICKEYBYTES as u32));
        msg.extend(&pubkey_bytes);
        try!(msg.write_u32::<BigEndian>(ed25519::SECRETKEYBYTES as u32));
        msg.extend(seckey_bytes.iter()); // LOL, there's no iterator for &[u8, 64] because 64 is a lot
        try!(msg.write_u32::<BigEndian>(comment.as_bytes().len() as u32));
        msg.extend(comment.as_bytes());
        Ok(SecStr::new(msg))
    } else { Err(Error::InappropriateFormat) }
}

#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))]
pub fn ssh_agent_send_message(msg: SecStr) -> Result<()> {
    if let Some(sock_path) = env::var_os("SSH_AUTH_SOCK") {
        let mut stream = try!(UnixStream::connect(sock_path));
        try!(stream.write_u32::<BigEndian>(msg.unsecure().len() as u32));
        try!(stream.write(msg.unsecure()));
        let mut response = vec![0; 5];
        try!(stream.read(&mut response));
        try!(stream.shutdown(Shutdown::Both));
        Ok(())
    } else { Err(Error::SSHAgentSocketNotFound) }
}

#[cfg(any(not(unix), target_os = "android", target_os = "ios"))]
pub fn ssh_agent_send_message(_: SecStr) -> Result<()> {
    Err(Error::NotAvailableOnPlatform)
}
<commit_msg>ssh tests<commit_after>use data::*;
use secstr::SecStr;
use rusterpassword::*;
use rustc_serialize::base64::{ToBase64, STANDARD};
use sodiumoxide::crypto::sign::ed25519;
use byteorder::{BigEndian, WriteBytesExt};
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use unix_socket::UnixStream;
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::env;
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::io::{Read, Write};
#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))] use std::net::Shutdown;

pub enum Output {
    PrivateText(SecStr),
    OpenText(String),
    PrivateBinary(SecStr),
    Ed25519Keypair(Ed25519Usage, ed25519::PublicKey, ed25519::SecretKey),
}

fn pick_tpl(tpl: &PasswordTemplate) -> &'static [&'static str] {
    match *tpl {
       PasswordTemplate::Maximum  => TEMPLATES_MAXIMUM,
       PasswordTemplate::Long     => TEMPLATES_LONG,
       PasswordTemplate::Medium   => TEMPLATES_MEDIUM,
       PasswordTemplate::Short    => TEMPLATES_SHORT,
       PasswordTemplate::Basic    => TEMPLATES_BASIC,
       PasswordTemplate::Pin      => TEMPLATES_PIN,
    }
}

pub fn process_output(entry_name: &str, master_key: &SecStr, field: &Field) -> Result<Output> {
    match *field {
        Field::Derived { counter, ref site_name, ref usage } => {
            let site_seed = try!(gen_site_seed(master_key, &site_name.clone().unwrap_or(entry_name.to_string()), counter)
                                 .map_err(|_| Error::SeedGenerationError));
            match *usage {
                DerivedUsage::Password(ref tpl) =>
                    Ok(Output::PrivateText(gen_site_password(&site_seed, pick_tpl(tpl)))),
                DerivedUsage::Ed25519Key(ref keyusage) => {
                    let edseed = try!(ed25519::Seed::from_slice(site_seed.unsecure()).ok_or(Error::WrongDerivedKeyLength));
                    let (pubkey, seckey) = ed25519::keypair_from_seed(&edseed);
                    Ok(Output::Ed25519Keypair(*keyusage, pubkey, seckey))
                },
                DerivedUsage::RawKey =>
                    Ok(Output::PrivateBinary(site_seed)),
            }
        },
        Field::Stored { ref data, ref usage } =>
            match *usage {
                StoredUsage::Password =>
                    Ok(Output::PrivateText(SecStr::new(data.unsecure().to_vec()))),
                StoredUsage::Text =>
                    Ok(Output::OpenText(try!(String::from_utf8(data.unsecure().to_vec()))))
            }
    }
}

pub fn ssh_public_key_output(keypair: &Output, comment: &str) -> Result<String> {
    if let &Output::Ed25519Keypair(Ed25519Usage::SSH, ed25519::PublicKey(pubkey_bytes), _) = keypair {
        let mut raw = vec![];
        try!(raw.write_u32::<BigEndian>(11));
        raw.extend(b"ssh-ed25519");
        try!(raw.write_u32::<BigEndian>(ed25519::PUBLICKEYBYTES as u32));
        raw.extend(&pubkey_bytes);
        Ok("ssh-ed25519 ".to_string() + &raw.to_base64(STANDARD) + " " + comment)
    } else { Err(Error::InappropriateFormat) }
}

pub fn ssh_private_key_agent_message(keypair: &Output, comment: &str) -> Result<SecStr> {
    if let &Output::Ed25519Keypair(Ed25519Usage::SSH, ed25519::PublicKey(pubkey_bytes), ed25519::SecretKey(seckey_bytes)) = keypair {
        let mut msg = vec![17u8];
        try!(msg.write_u32::<BigEndian>(11));
        msg.extend(b"ssh-ed25519");
        try!(msg.write_u32::<BigEndian>(ed25519::PUBLICKEYBYTES as u32));
        msg.extend(&pubkey_bytes);
        try!(msg.write_u32::<BigEndian>(ed25519::SECRETKEYBYTES as u32));
        msg.extend(seckey_bytes.iter()); // LOL, there's no iterator for &[u8, 64] because 64 is a lot
        try!(msg.write_u32::<BigEndian>(comment.as_bytes().len() as u32));
        msg.extend(comment.as_bytes());
        Ok(SecStr::new(msg))
    } else { Err(Error::InappropriateFormat) }
}

#[cfg(all(unix, not(target_os = "android"), not(target_os = "ios")))]
pub fn ssh_agent_send_message(msg: SecStr) -> Result<()> {
    if let Some(sock_path) = env::var_os("SSH_AUTH_SOCK") {
        let mut stream = try!(UnixStream::connect(sock_path));
        try!(stream.write_u32::<BigEndian>(msg.unsecure().len() as u32));
        try!(stream.write(msg.unsecure()));
        let mut response = vec![0; 5];
        try!(stream.read(&mut response));
        try!(stream.shutdown(Shutdown::Both));
        Ok(())
    } else { Err(Error::SSHAgentSocketNotFound) }
}

#[cfg(any(not(unix), target_os = "android", target_os = "ios"))]
pub fn ssh_agent_send_message(_: SecStr) -> Result<()> {
    Err(Error::NotAvailableOnPlatform)
}

#[cfg(test)]
mod tests {
    use super::*;
    use data::*;
    use sodiumoxide::crypto::sign::ed25519;

    fn keypair() -> Output {
        let (pubkey, seckey) = ed25519::keypair_from_seed(&ed25519::Seed::from_slice(b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap());
        Output::Ed25519Keypair(Ed25519Usage::SSH, pubkey, seckey)
    }

    #[test]
    fn test_ssh_public_key_output() {
        assert_eq!(ssh_public_key_output(&keypair(), "myComment").unwrap(), "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK8Go+MpFxTk81bBnJsVzRlR7G5mYqp3vgdUfyiTgzQd myComment");
    }

    #[test]
    fn test_ssh_private_key_agent_message() {
        let msg = ssh_private_key_agent_message(&keypair(), "myComment").unwrap();
        assert_eq!(Vec::from(msg.unsecure()), vec![17, 0, 0, 0, 11, 115, 115, 104, 45, 101, 100, 50, 53, 53, 49, 57, 0, 0, 0, 32, 175, 6, 163, 227, 41, 23, 20, 228, 243, 86, 193, 156, 155, 21, 205, 25, 81, 236, 110, 102, 98, 170, 119, 190, 7, 84, 127, 40, 147, 131, 52, 29, 0, 0, 0, 64, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 175, 6, 163, 227, 41, 23, 20, 228, 243, 86, 193, 156, 155, 21, 205, 25, 81, 236, 110, 102, 98, 170, 119, 190, 7, 84, 127, 40, 147, 131, 52, 29, 0, 0, 0, 9, 109, 121, 67, 111, 109, 109, 101, 110, 116]);
    }
}
<|endoftext|>


<commit_before>use super::*;

#[test]
fn test_struct_info_roundtrip() {
    let s = ItemEnum::Struct(Struct {
        struct_type: StructType::Plain,
        generics: Generics {
            params: vec![],
            where_predicates: vec![]
        },
        fields_stripped: false,
        fields: vec![],
        impls: vec![],
    });

    let struct_json = serde_json::to_string(&s).unwrap();

    let de_s = serde_json::from_str(&struct_json).unwrap();

    assert_eq!(s, de_s);
}

#[test]
fn test_union_info_roundtrip() {
    let u = ItemEnum::Union(Union {
        generics: Generics {
            params: vec![],
            where_predicates: vec![]
        },
        fields_stripped: false,
        fields: vec![],
        impls: vec![],
    });

    let union_json = serde_json::to_string(&u).unwrap();

    let de_u = serde_json::from_str(&union_json).unwrap();

    assert_eq!(u, de_u);
}<commit_msg>x.py fmt<commit_after>use super::*;

#[test]
fn test_struct_info_roundtrip() {
    let s = ItemEnum::Struct(Struct {
        struct_type: StructType::Plain,
        generics: Generics { params: vec![], where_predicates: vec![] },
        fields_stripped: false,
        fields: vec![],
        impls: vec![],
    });

    let struct_json = serde_json::to_string(&s).unwrap();

    let de_s = serde_json::from_str(&struct_json).unwrap();

    assert_eq!(s, de_s);
}

#[test]
fn test_union_info_roundtrip() {
    let u = ItemEnum::Union(Union {
        generics: Generics { params: vec![], where_predicates: vec![] },
        fields_stripped: false,
        fields: vec![],
        impls: vec![],
    });

    let union_json = serde_json::to_string(&u).unwrap();

    let de_u = serde_json::from_str(&union_json).unwrap();

    assert_eq!(u, de_u);
}
<|endoftext|>


<commit_before>use std::result::Result;

use silkthread_base::*;
use silkthread_base::basic::*;

use super::super::lex::lex_tool::*;
use super::script::*;

///
/// Represents a parse error
///
#[derive(Serialize, Deserialize, Debug)]
pub struct ParseError {

}

///
/// Tool that parses our scripting language
///
pub struct ParseScriptTool {
}

impl ParseError {
    pub fn new() -> ParseError {
        ParseError { }
    }
}

struct ParseState<'a> {
    remaining: &'a [ScriptToken]
}

impl<'a> ParseState<'a> {
    ///
    /// Returns true if a token is considered syntax (gets returned from lookahead)
    ///
    fn is_syntax(token: &ScriptToken) -> bool {
        match token.token {
            ScriptLexerToken::Whitespace    |
            ScriptLexerToken::Comment       => false,

            _                               => true
        }
    }

    ///
    /// Looks ahead to the next syntactically relevant lexer match (and returns the tokens after it)
    ///
    fn lookahead(&self) -> Option<(&'a ScriptToken, &'a [ScriptToken])> {
        let mut index   = 0;
        let len         = self.remaining.len();

        loop {
            if index >= len {
                return None;
            } else if ParseState::is_syntax(&self.remaining[index]) {
                let token       = &self.remaining[index];
                let remaining   = &self.remaining[index+1..len];
                return Some((token, remaining));
            }

            index += 1;
        }
    }

    ///
    /// If the next token matches the specified token, consumes it and returns
    /// its content.
    ///
    fn lookahead_is(&self, token: ScriptLexerToken) -> bool {
        if let Some((lookahead, _remaining)) = self.lookahead() {
            if lookahead.token == token {
                // Token matches
                true
            } else {
                // Token does not match
                false
            }
        } else {
            // End of file
            if token == ScriptLexerToken::EndOfFile {
                // With lookahead_is we can look for the end of file (but we can't accept it because there's no token data associated with it)
                true
            } else {
                false
            }
        }
    }

    ///
    /// If the next token matches the specified token, consumes it and returns
    /// its content.
    ///
    fn accept(&mut self, token: ScriptLexerToken) -> Option<&'a ScriptToken> {
        if let Some((lookahead, remaining)) = self.lookahead() {
            if lookahead.token == token {
                // Token matches: remove it from the input and return it
                self.remaining = remaining;
                Some(lookahead)
            } else {
                // Next token does not match
                None
            }
        } else {
            // Reached the end of file
            None
        }
    }

    ///
    /// Parses a statement
    ///
    pub fn parse_statement(&mut self) -> Result<Script, ParseError> {
        if self.accept(ScriptLexerToken::Newline).is_some() {
            // Newlines are ignored
            self.parse_statement()

        } else if self.accept(ScriptLexerToken::Let).is_some() {
            // let identifier = expression
            self.parse_let()

        } else if self.accept(ScriptLexerToken::Var).is_some() {
            // var identifier = expression
            self.parse_var()

        } else if self.accept(ScriptLexerToken::Def).is_some() {
            // def fn args { statements }
            self.parse_def()

        } else if self.accept(ScriptLexerToken::If).is_some() {
            // if expression { statements }
            self.parse_if()

        } else if self.accept(ScriptLexerToken::Using).is_some() {
            // using expression { statements }
            self.parse_using()

        } else if self.accept(ScriptLexerToken::While).is_some() {
            // while expression { statements }
            self.parse_while()

        } else if self.accept(ScriptLexerToken::Loop).is_some() {
            // loop { statements }
            self.parse_loop()

        } else if self.accept(ScriptLexerToken::For).is_some() {
            // for identifier in expression { statements }
            self.parse_for()

        } else if self.lookahead_is(ScriptLexerToken::Identifier) {
            // While commands are either <Expression> or <Expression> <Expression>, we
            // force the first expression to be an identifier at the moment
            self.parse_command()

        } else {
            // Unrecognised token
            Err(ParseError::new())
        }
    }

    ///
    /// Parses a command
    ///
    /// Syntax '<expression>', '<expression> <expression>'
    ///
    pub fn parse_command(&mut self) -> Result<Script, ParseError> {
        // Starts with an expression specifying the command to run
        self.parse_expression().and_then(move |command_expression| {
            // Followed by arguments (or an end-of-expression marker)
            if self.accept(ScriptLexerToken::Newline).is_some()
               || self.lookahead_is(ScriptLexerToken::EndOfFile) {
                // Newline or EOF ends a command
                Ok(Script::RunCommand(command_expression, None))

            } else {
                // Anything else should be an argument expression
                self.parse_expression().and_then(move |argument_expression| {
                    Ok(Script::RunCommand(command_expression, Some(argument_expression)))

                }).and_then(move |command| {
                    // Command must be followed by a newline
                    if self.accept(ScriptLexerToken::Newline).is_some()
                       || self.lookahead_is(ScriptLexerToken::EndOfFile) {
                        Ok(command)
                    } else {
                        Err(ParseError::new())
                    }

                })

            }
        })
    }

    ///
    /// Parses an Expression
    ///
    pub fn parse_expression(&mut self) -> Result<Expression, ParseError> {
        // Get the LHS of the expression
        let left_expr = self.parse_simple_expression();

        // Look for a RHS
        left_expr.and_then(|left_expr| self.parse_expression_rhs(left_expr))
    }

    ///
    /// Parses the RHS of an expression (if there is one)
    ///
    pub fn parse_expression_rhs(&mut self, left_expr: Expression) -> Result<Expression, ParseError> {
        if self.accept(ScriptLexerToken::Symbol(String::from("."))).is_some() {
            // 'a.b' field access
            let right_expr = self.parse_expression();

            right_expr.map(|right_expr| Expression::FieldAccess(Box::new((left_expr, right_expr))))

        } else if self.accept(ScriptLexerToken::Symbol(String::from("["))).is_some() {
            // a[b] indexing
            let right_expr = self.parse_expression();

            right_expr.and_then(|right_expr| {
                if self.accept(ScriptLexerToken::Symbol(String::from("]"))).is_some() {
                    // Got all of a[b]
                    Ok(Expression::Index(Box::new((left_expr, right_expr))))
                } else {
                    // Missing ']'
                    Err(ParseError::new())
                }
            })

        } else {
            Ok(left_expr)

        }
    }

    ///
    /// Parses a simple expression
    ///
    pub fn parse_simple_expression(&mut self) -> Result<Expression, ParseError> {
        if self.accept(ScriptLexerToken::Newline).is_some() {
            // Ignore newlines within an expression
            self.parse_expression()

        } else if let Some(identifier) = self.accept(ScriptLexerToken::Identifier) {
            // Simple expression
            Ok(Expression::Identifier(identifier.clone()))

        } else if let Some(number) = self.accept(ScriptLexerToken::Number) {
            // Simple expression
            Ok(Expression::Number(number.clone()))

        } else if let Some(number) = self.accept(ScriptLexerToken::HexNumber) {
            // Hex numbers work like normal numbers
            Ok(Expression::Number(number.clone()))

        } else if let Some(string) = self.accept(ScriptLexerToken::String) {
            // Simple expression
            Ok(Expression::String(string.clone()))

        } else {
            // Syntax error
            Err(ParseError::new())

        }
    }

    pub fn parse_let(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_var(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_def(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_if(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_using(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_while(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_loop(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_for(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }
}

impl ParseScriptTool {
    ///
    /// Creates a new tool from the parse script tool
    ///
    pub fn new_tool() -> Box<Tool> {
        Box::new(make_tool(move |script: Vec<LexerMatch>| -> Result<Vec<Script>, ParseError> {
            ParseScriptTool::parse(&script)
        }))
    }

    ///
    /// Tries to parse a script from the output of the lexer
    ///
    pub fn parse(input: &[LexerMatch]) -> Result<Vec<Script>, ParseError> {
        // Convert to script tokens
        let as_script_token: Vec<ScriptToken> = input
            .iter()
            .map(|token| ScriptToken::from_lexer_match(token))
            .collect();

        // Parse until we reach the end of the file
        let mut parser = ParseState { remaining: &as_script_token };
        let mut result = vec![];

        while !parser.lookahead_is(ScriptLexerToken::EndOfFile) {
            let next_statement = parser.parse_statement();

            match next_statement {
                // Fail out if we get a parse failure
                Err(failure)        => return Err(failure),

                // Build out the result otherwise
                Ok(next_statement)  => result.push(next_statement)
            }

            // Swallow any trailing newlines
            while parser.accept(ScriptLexerToken::Newline).is_some() { }
        }

        Ok(result)
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::lex_script_tool::*;

    ///
    /// Performs lexing
    ///
    fn lex(text: &str) -> Vec<LexerMatch> {
        let lexer = create_lex_script_tool();

        lexer.lex(text)
    }

    fn parse(text: &str) -> Result<Vec<Script>, ParseError> {
        let lexed = lex(text);
        ParseScriptTool::parse(&lexed)
    }

    #[test]
    fn can_parse_command_statement() {
        let statement   = "some-command";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Identifier(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_field_access() {
        let statement   = "some-command.some-field";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::FieldAccess(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_array_indexing() {
        let statement   = "some-command[0]";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Index(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_command_statement_with_parameter() {
        let statement   = "some-command 1";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Identifier(_), Some(Expression::Number(_))) => true, _ => false});
    }

    #[test]
    fn cannot_try_to_put_multiple_statements_on_one_line() {
        let statement   = "some-command 1 some-other-command";
        let parsed      = parse(statement);

        assert!(parsed.is_err());
    }

    #[test]
    fn can_parse_multiple_lines() {
        let statement   = "some-command\nsome-other-command";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());
        assert!(parsed.unwrap().len() == 2);
    }
}
<commit_msg>Start in on array expressions<commit_after>use std::result::Result;

use silkthread_base::*;
use silkthread_base::basic::*;

use super::super::lex::lex_tool::*;
use super::script::*;

///
/// Represents a parse error
///
#[derive(Serialize, Deserialize, Debug)]
pub struct ParseError {

}

///
/// Tool that parses our scripting language
///
pub struct ParseScriptTool {
}

impl ParseError {
    pub fn new() -> ParseError {
        ParseError { }
    }
}

struct ParseState<'a> {
    remaining: &'a [ScriptToken]
}

impl<'a> ParseState<'a> {
    ///
    /// Returns true if a token is considered syntax (gets returned from lookahead)
    ///
    fn is_syntax(token: &ScriptToken) -> bool {
        match token.token {
            ScriptLexerToken::Whitespace    |
            ScriptLexerToken::Comment       => false,

            _                               => true
        }
    }

    ///
    /// Looks ahead to the next syntactically relevant lexer match (and returns the tokens after it)
    ///
    fn lookahead(&self) -> Option<(&'a ScriptToken, &'a [ScriptToken])> {
        let mut index   = 0;
        let len         = self.remaining.len();

        loop {
            if index >= len {
                return None;
            } else if ParseState::is_syntax(&self.remaining[index]) {
                let token       = &self.remaining[index];
                let remaining   = &self.remaining[index+1..len];
                return Some((token, remaining));
            }

            index += 1;
        }
    }

    ///
    /// If the next token matches the specified token, consumes it and returns
    /// its content.
    ///
    fn lookahead_is(&self, token: ScriptLexerToken) -> bool {
        if let Some((lookahead, _remaining)) = self.lookahead() {
            if lookahead.token == token {
                // Token matches
                true
            } else {
                // Token does not match
                false
            }
        } else {
            // End of file
            if token == ScriptLexerToken::EndOfFile {
                // With lookahead_is we can look for the end of file (but we can't accept it because there's no token data associated with it)
                true
            } else {
                false
            }
        }
    }

    ///
    /// If the next token matches the specified token, consumes it and returns
    /// its content.
    ///
    fn accept(&mut self, token: ScriptLexerToken) -> Option<&'a ScriptToken> {
        if let Some((lookahead, remaining)) = self.lookahead() {
            if lookahead.token == token {
                // Token matches: remove it from the input and return it
                self.remaining = remaining;
                Some(lookahead)
            } else {
                // Next token does not match
                None
            }
        } else {
            // Reached the end of file
            None
        }
    }

    ///
    /// Parses a statement
    ///
    pub fn parse_statement(&mut self) -> Result<Script, ParseError> {
        if self.accept(ScriptLexerToken::Newline).is_some() {
            // Newlines are ignored
            self.parse_statement()

        } else if self.accept(ScriptLexerToken::Let).is_some() {
            // let identifier = expression
            self.parse_let()

        } else if self.accept(ScriptLexerToken::Var).is_some() {
            // var identifier = expression
            self.parse_var()

        } else if self.accept(ScriptLexerToken::Def).is_some() {
            // def fn args { statements }
            self.parse_def()

        } else if self.accept(ScriptLexerToken::If).is_some() {
            // if expression { statements }
            self.parse_if()

        } else if self.accept(ScriptLexerToken::Using).is_some() {
            // using expression { statements }
            self.parse_using()

        } else if self.accept(ScriptLexerToken::While).is_some() {
            // while expression { statements }
            self.parse_while()

        } else if self.accept(ScriptLexerToken::Loop).is_some() {
            // loop { statements }
            self.parse_loop()

        } else if self.accept(ScriptLexerToken::For).is_some() {
            // for identifier in expression { statements }
            self.parse_for()

        } else if self.lookahead_is(ScriptLexerToken::Identifier) {
            // While commands are either <Expression> or <Expression> <Expression>, we
            // force the first expression to be an identifier at the moment
            self.parse_command()

        } else {
            // Unrecognised token
            Err(ParseError::new())
        }
    }

    ///
    /// Parses a command
    ///
    /// Syntax '<expression>', '<expression> <expression>'
    ///
    pub fn parse_command(&mut self) -> Result<Script, ParseError> {
        // Starts with an expression specifying the command to run
        self.parse_expression().and_then(move |command_expression| {
            // Followed by arguments (or an end-of-expression marker)
            if self.accept(ScriptLexerToken::Newline).is_some()
               || self.lookahead_is(ScriptLexerToken::EndOfFile) {
                // Newline or EOF ends a command
                Ok(Script::RunCommand(command_expression, None))

            } else {
                // Anything else should be an argument expression
                self.parse_expression().and_then(move |argument_expression| {
                    Ok(Script::RunCommand(command_expression, Some(argument_expression)))

                }).and_then(move |command| {
                    // Command must be followed by a newline
                    if self.accept(ScriptLexerToken::Newline).is_some()
                       || self.lookahead_is(ScriptLexerToken::EndOfFile) {
                        Ok(command)
                    } else {
                        Err(ParseError::new())
                    }

                })

            }
        })
    }

    ///
    /// Parses an Expression
    ///
    pub fn parse_expression(&mut self) -> Result<Expression, ParseError> {
        let left_expr = if self.lookahead_is(ScriptLexerToken::Symbol(String::from("["))) {
            self.parse_array_expression()

        } else {
            // Simple expression
            self.parse_simple_expression()

        };

        // Look for a RHS
        left_expr.and_then(|left_expr| self.parse_expression_rhs(left_expr))
    }

    ///
    /// Parses the RHS of an expression (if there is one)
    ///
    pub fn parse_expression_rhs(&mut self, left_expr: Expression) -> Result<Expression, ParseError> {
        if self.accept(ScriptLexerToken::Symbol(String::from("."))).is_some() {
            // 'a.b' field access
            let right_expr = self.parse_expression();

            right_expr.map(|right_expr| Expression::FieldAccess(Box::new((left_expr, right_expr))))

        } else if self.accept(ScriptLexerToken::Symbol(String::from("["))).is_some() {
            // a[b] indexing
            let right_expr = self.parse_expression();

            right_expr.and_then(|right_expr| {
                if self.accept(ScriptLexerToken::Symbol(String::from("]"))).is_some() {
                    // Got all of a[b]
                    Ok(Expression::Index(Box::new((left_expr, right_expr))))
                } else {
                    // Missing ']'
                    Err(ParseError::new())
                }
            })

        } else {
            Ok(left_expr)

        }
    }

    ///
    /// Parses an array expression
    ///
    pub fn parse_array_expression(&mut self) -> Result<Expression, ParseError> {
        unimplemented!()
    }

    ///
    /// Parses a simple expression
    ///
    pub fn parse_simple_expression(&mut self) -> Result<Expression, ParseError> {
        if self.accept(ScriptLexerToken::Newline).is_some() {
            // Ignore newlines within an expression
            self.parse_expression()

        } else if let Some(identifier) = self.accept(ScriptLexerToken::Identifier) {
            // Simple expression
            Ok(Expression::Identifier(identifier.clone()))

        } else if let Some(number) = self.accept(ScriptLexerToken::Number) {
            // Simple expression
            Ok(Expression::Number(number.clone()))

        } else if let Some(number) = self.accept(ScriptLexerToken::HexNumber) {
            // Hex numbers work like normal numbers
            Ok(Expression::Number(number.clone()))

        } else if let Some(string) = self.accept(ScriptLexerToken::String) {
            // Simple expression
            Ok(Expression::String(string.clone()))

        } else {
            // Syntax error
            Err(ParseError::new())

        }
    }

    pub fn parse_let(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_var(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_def(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_if(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_using(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_while(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_loop(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }

    pub fn parse_for(&mut self) -> Result<Script, ParseError> {
        unimplemented!()
    }
}

impl ParseScriptTool {
    ///
    /// Creates a new tool from the parse script tool
    ///
    pub fn new_tool() -> Box<Tool> {
        Box::new(make_tool(move |script: Vec<LexerMatch>| -> Result<Vec<Script>, ParseError> {
            ParseScriptTool::parse(&script)
        }))
    }

    ///
    /// Tries to parse a script from the output of the lexer
    ///
    pub fn parse(input: &[LexerMatch]) -> Result<Vec<Script>, ParseError> {
        // Convert to script tokens
        let as_script_token: Vec<ScriptToken> = input
            .iter()
            .map(|token| ScriptToken::from_lexer_match(token))
            .collect();

        // Parse until we reach the end of the file
        let mut parser = ParseState { remaining: &as_script_token };
        let mut result = vec![];

        while !parser.lookahead_is(ScriptLexerToken::EndOfFile) {
            let next_statement = parser.parse_statement();

            match next_statement {
                // Fail out if we get a parse failure
                Err(failure)        => return Err(failure),

                // Build out the result otherwise
                Ok(next_statement)  => result.push(next_statement)
            }

            // Swallow any trailing newlines
            while parser.accept(ScriptLexerToken::Newline).is_some() { }
        }

        Ok(result)
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::lex_script_tool::*;

    ///
    /// Performs lexing
    ///
    fn lex(text: &str) -> Vec<LexerMatch> {
        let lexer = create_lex_script_tool();

        lexer.lex(text)
    }

    fn parse(text: &str) -> Result<Vec<Script>, ParseError> {
        let lexed = lex(text);
        ParseScriptTool::parse(&lexed)
    }

    #[test]
    fn can_parse_command_statement() {
        let statement   = "some-command";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Identifier(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_field_access() {
        let statement   = "some-command.some-field";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::FieldAccess(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_array_indexing() {
        let statement   = "some-command[0]";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Index(_), None) => true, _ => false});
    }

    #[test]
    fn can_parse_command_statement_with_parameter() {
        let statement   = "some-command 1";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Identifier(_), Some(Expression::Number(_))) => true, _ => false});
    }

    #[test]
    fn can_parse_command_statement_with_array_parameter() {
        let statement   = "some-command [ 1, 2, 3 ]";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());

        let result = parsed.unwrap();
        assert!(result.len() == 1);

        let ref cmd = result[0];
        assert!(match cmd { &Script::RunCommand(Expression::Identifier(_), Some(Expression::Array(_))) => true, _ => false});
    }

    #[test]
    fn cannot_try_to_put_multiple_statements_on_one_line() {
        let statement   = "some-command 1 some-other-command";
        let parsed      = parse(statement);

        assert!(parsed.is_err());
    }

    #[test]
    fn can_parse_multiple_lines() {
        let statement   = "some-command\nsome-other-command";
        let parsed      = parse(statement);

        assert!(parsed.is_ok());
        assert!(parsed.unwrap().len() == 2);
    }
}
<|endoftext|>


<commit_before>use num::{self};
use num::traits::{Zero, WrappingAdd, WrappingSub, WrappingMul, CheckedShl, CheckedShr, Signed};
use std::mem;
use std::cmp::{min};
use std::ops::{Add, Sub, Mul, AddAssign, SubAssign, MulAssign, Shl, Shr, Neg, Index};
use std::fmt::Debug;

pub trait Integer:
  'static + num::PrimInt + num::Integer + num::FromPrimitive +
  AddAssign <Self> + SubAssign <Self> + MulAssign <Self> +
  WrappingAdd + WrappingSub + WrappingMul +
  for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + for<'a> Mul <&'a Self, Output = Self> +
  CheckedShl + CheckedShr + Shl <u32, Output = Self> + Shr <u32, Output = Self> +
  Debug {}
impl <T: 'static + num::PrimInt + num::Integer + num::FromPrimitive + AddAssign <Self> + SubAssign <Self> + MulAssign <Self> + WrappingAdd + WrappingSub + WrappingMul + for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + for<'a> Mul <&'a Self, Output = Self> + CheckedShl + CheckedShr + Shl <u32, Output = Self> + Shr <u32, Output = Self> + Debug> Integer for T {}

pub trait HasCoordinates
   {
  type Coordinate: Integer + Signed + Vector + HasCoordinates <Coordinate = Self::Coordinate>;}

pub trait Vector:
  'static + Sized + Copy + Clone + HasCoordinates +
  Add <Self, Output = Self> + Sub <Self, Output = Self> + Mul <<Self as HasCoordinates>::Coordinate, Output = Self> +
  for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + //for<'a> Mul <&'a <Self as HasCoordinates>::Coordinate, Output = Self> +
  AddAssign <Self> + SubAssign <Self> + MulAssign <<Self as HasCoordinates>::Coordinate> +
  for<'a> AddAssign <&'a Self> + for<'a> SubAssign <&'a Self> + //for<'a> MulAssign <&'a <Self as HasCoordinates>::Coordinate> +
  Zero + Neg <Output = Self>
  {
  const DIMENSIONS: usize;
  fn coordinate (&self, which: usize)->Self::Coordinate;
  fn set_coordinate (&mut self, which: usize, value: Self::Coordinate);
}

pub mod impls {
  use super::*;
  use nalgebra::*;
  macro_rules! impl_vector {
    ($([$coordinates: expr, $Vector: ident],)*) => {
      $(
        impl <T: Integer + Signed + Vector + HasCoordinates <Coordinate = T>> HasCoordinates for $Vector <T> {type Coordinate = T;}
        impl <T: Integer + Signed + Vector + HasCoordinates <Coordinate = T>> Vector for $Vector <T> {
          const DIMENSIONS: usize = $coordinates;
          fn coordinate (&self, which: usize)->Self::Coordinate {self [which]}
          fn set_coordinate (&mut self, which: usize, value: Self::Coordinate) {self [which] = value}
        }
      )*
    }
  }
  macro_rules! impl_integer {
    ($($Integer: ident,)*) => {
      $(
        impl HasCoordinates for $Integer {type Coordinate = $Integer;}
        impl Vector for $Integer {
          const DIMENSIONS: usize = 1;
          fn coordinate (&self, which: usize)->Self::Coordinate {*self}
          fn set_coordinate (&mut self, which: usize, value: Self::Coordinate) {*self = value}
        }
      )*
    }
  }
  impl_vector! ([1, Vector1], [2, Vector2], [3, Vector3], [4, Vector4], [5, Vector5], [6, Vector6],);
  impl_integer! (i8, i16, i32, i64, isize,);
}

/// Right-shift an integer, but round to nearest, with ties rounding to even.
///
/// This minimizes error, and avoids a directional bias.
pub fn shr_nicely_rounded <T: Integer> (input: T, shift: u32)->T {
  if shift == 0 {return input}
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let half_shift = shift - 1;
  let half = T::one() << half_shift;
  let mask = divisor.wrapping_sub (&T::one());
  if (input & mask) == half {
    let shifted = input >> shift;
    shifted + (shifted & T::one())
  } else {
    (input >> shift) + ((input & half) >> half_shift)
  }
}

/// Right-shift an integer, but round to even.
///
/// This avoids a directional bias.
pub fn shr_round_to_even <T: Integer> (input: T, shift: u32)->T {
  if shift == 0 {return input}
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let mask = divisor.wrapping_sub (&T::one());
  let value = input >> shift;
  if (input & mask) == T::zero() {
    value
  }
  else {
    value + (value & T::one())
  }
}

/// Right-shift an integer, but round towards positive infinity.
pub fn shr_ceil <T: Integer> (input: T, shift: u32)->T {
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let mask = divisor.wrapping_sub (&T::one());
  (input >> shift) + if input & mask != T::zero() {T::one()} else {T::zero()}
}

/// Left-shift an integer, returning Some(input*(2^shift)) if it fits within the type, None otherwise.
pub fn overflow_checked_shl <T: Integer> (input: T, shift: u32)->Option <T> {
  if input == T::zero() {return Some (T::zero())}
  let maximum = match T::max_value().checked_shr (shift) {
    None => return None,
    Some (value) => value,
  };
  if input > maximum {return None}
  let minimum = T::min_value() >> shift;
  if input < minimum {return None}
  Some (input << shift)
}

/// Compute the arithmetic mean of two integers, rounded towards negative infinity. Never overflows.
pub fn mean_floor <T: Integer> (first: T, second: T)->T {
  (first >> 1u32) + (second >> 1u32) + (first & second & T::one())
}

/// Compute the arithmetic mean of two integers, rounded towards positive infinity. Never overflows.
pub fn mean_ceil <T: Integer> (first: T, second: T)->T {
  (first >> 1u32) + (second >> 1u32) + ((first | second) & T::one())
}

pub mod polynomial;

#[cfg (test)]
mod tests {
  use super::*;
  use num::{One, ToPrimitive, Integer};
  use num::bigint::BigInt;
  use num::rational::{Ratio};
  use std::cmp::Ordering;

  fn perfect_shr_nicely_rounded <T: Integer> (input: T, shift: u32)->BigInt where BigInt: From <T> {
    let perfect_result = Ratio::new (BigInt::from (input), BigInt::one() << shift as usize);
    let rounded_down = perfect_result.floor();
    let fraction = & perfect_result - & rounded_down;
    let rounded_down = rounded_down.to_integer();
    match fraction.cmp (& Ratio::new (BigInt::one(), BigInt::one() << 1)) {
      Ordering::Less => rounded_down, Ordering::Greater => rounded_down + BigInt::one() ,
      Ordering::Equal => & rounded_down + rounded_down.mod_floor (& (BigInt::one() << 1)),
    }
  }

  fn perfect_shr_round_to_even <T: Integer> (input: T, shift: u32)->BigInt where BigInt: From <T> {
    let perfect_result = Ratio::new (BigInt::from (input), BigInt::one() << shift as usize);
    let rounded_down = perfect_result.floor();
    if perfect_result == rounded_down {rounded_down.to_integer()}
    else {
      let rounded_down = rounded_down.to_integer();
      if &rounded_down >> 1 << 1 == rounded_down {
        rounded_down
      }
      else {rounded_down + 1}
    }
  }

  #[test]
  fn test_shr_nicely_rounded() {
    let inputs: Vec<(i64, u32, i64)> = vec![
      (0, 0, 0), (0, 5, 0), (1, 3, 0), (4, 3, 0), (5, 3, 1),
      (999, 1, 500), (998, 1, 499), (997, 1, 498)
    ];
    for (input, shift, result) in inputs {
      assert_eq!(shr_nicely_rounded (input, shift), result);
      assert_eq!(shr_nicely_rounded (-input, shift), -result);
    }
  }

  #[test]
  fn test_shr_round_to_even() {
    let inputs: Vec<(i64, u32, i64)> = vec![
      (0, 0, 0), (0, 5, 0), (1, 3, 0), (4, 3, 0), (5, 3, 0),
      (999, 1, 500), (998, 1, 499), (997, 1, 498)
    ];
    for (input, shift, result) in inputs {
      assert_eq!(shr_round_to_even (input, shift), result);
      assert_eq!(shr_round_to_even (-input, shift), -result);
    }
  }

  quickcheck! {
    fn quickcheck_shr_nicely_rounded_signed (input: i32, shift: u8)->bool {
      let result = shr_nicely_rounded (input, shift as u32);
      let perfect_result = perfect_shr_nicely_rounded (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_nicely_rounded_unsigned (input: u32, shift: u8)->bool {
      let result = shr_nicely_rounded (input, shift as u32);
      let perfect_result = perfect_shr_nicely_rounded (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_round_to_even_signed (input: i32, shift: u8)->bool {
      let result = shr_round_to_even (input, shift as u32);
      let perfect_result = perfect_shr_round_to_even (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_round_to_even_unsigned (input: u32, shift: u8)->bool {
      let result = shr_round_to_even (input, shift as u32);
      let perfect_result = perfect_shr_round_to_even (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_overflow_checked_shl (input: i32, shift: u8)->bool {
      let result = overflow_checked_shl (input, shift as u32);
      let perfect_result = BigInt::from (input) << shift as usize;
      result == perfect_result.to_i32()
    }
  }
}
<commit_msg>Reordered shr_nicely_rounded and shr_round_to_even so that their code is more similar<commit_after>use num::{self};
use num::traits::{Zero, WrappingAdd, WrappingSub, WrappingMul, CheckedShl, CheckedShr, Signed};
use std::mem;
use std::cmp::{min};
use std::ops::{Add, Sub, Mul, AddAssign, SubAssign, MulAssign, Shl, Shr, Neg, Index};
use std::fmt::Debug;

pub trait Integer:
  'static + num::PrimInt + num::Integer + num::FromPrimitive +
  AddAssign <Self> + SubAssign <Self> + MulAssign <Self> +
  WrappingAdd + WrappingSub + WrappingMul +
  for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + for<'a> Mul <&'a Self, Output = Self> +
  CheckedShl + CheckedShr + Shl <u32, Output = Self> + Shr <u32, Output = Self> +
  Debug {}
impl <T: 'static + num::PrimInt + num::Integer + num::FromPrimitive + AddAssign <Self> + SubAssign <Self> + MulAssign <Self> + WrappingAdd + WrappingSub + WrappingMul + for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + for<'a> Mul <&'a Self, Output = Self> + CheckedShl + CheckedShr + Shl <u32, Output = Self> + Shr <u32, Output = Self> + Debug> Integer for T {}

pub trait HasCoordinates
   {
  type Coordinate: Integer + Signed + Vector + HasCoordinates <Coordinate = Self::Coordinate>;}

pub trait Vector:
  'static + Sized + Copy + Clone + HasCoordinates +
  Add <Self, Output = Self> + Sub <Self, Output = Self> + Mul <<Self as HasCoordinates>::Coordinate, Output = Self> +
  for<'a> Add <&'a Self, Output = Self> + for<'a> Sub <&'a Self, Output = Self> + //for<'a> Mul <&'a <Self as HasCoordinates>::Coordinate, Output = Self> +
  AddAssign <Self> + SubAssign <Self> + MulAssign <<Self as HasCoordinates>::Coordinate> +
  for<'a> AddAssign <&'a Self> + for<'a> SubAssign <&'a Self> + //for<'a> MulAssign <&'a <Self as HasCoordinates>::Coordinate> +
  Zero + Neg <Output = Self>
  {
  const DIMENSIONS: usize;
  fn coordinate (&self, which: usize)->Self::Coordinate;
  fn set_coordinate (&mut self, which: usize, value: Self::Coordinate);
}

pub mod impls {
  use super::*;
  use nalgebra::*;
  macro_rules! impl_vector {
    ($([$coordinates: expr, $Vector: ident],)*) => {
      $(
        impl <T: Integer + Signed + Vector + HasCoordinates <Coordinate = T>> HasCoordinates for $Vector <T> {type Coordinate = T;}
        impl <T: Integer + Signed + Vector + HasCoordinates <Coordinate = T>> Vector for $Vector <T> {
          const DIMENSIONS: usize = $coordinates;
          fn coordinate (&self, which: usize)->Self::Coordinate {self [which]}
          fn set_coordinate (&mut self, which: usize, value: Self::Coordinate) {self [which] = value}
        }
      )*
    }
  }
  macro_rules! impl_integer {
    ($($Integer: ident,)*) => {
      $(
        impl HasCoordinates for $Integer {type Coordinate = $Integer;}
        impl Vector for $Integer {
          const DIMENSIONS: usize = 1;
          fn coordinate (&self, which: usize)->Self::Coordinate {*self}
          fn set_coordinate (&mut self, which: usize, value: Self::Coordinate) {*self = value}
        }
      )*
    }
  }
  impl_vector! ([1, Vector1], [2, Vector2], [3, Vector3], [4, Vector4], [5, Vector5], [6, Vector6],);
  impl_integer! (i8, i16, i32, i64, isize,);
}

/// Right-shift an integer, but round to nearest, with ties rounding to even.
///
/// This minimizes error, and avoids a directional bias.
pub fn shr_nicely_rounded <T: Integer> (input: T, shift: u32)->T {
  if shift == 0 {return input}
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let mask = divisor.wrapping_sub (&T::one());
  let shifted = input >> shift;
  let half_shift = shift - 1;
  let half = T::one() << half_shift;
  if (input & mask) == half {
    shifted + (shifted & T::one())
  } else {
    shifted + ((input & half) >> half_shift)
  }
}

/// Right-shift an integer, but round to even.
///
/// This avoids a directional bias.
pub fn shr_round_to_even <T: Integer> (input: T, shift: u32)->T {
  if shift == 0 {return input}
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let mask = divisor.wrapping_sub (&T::one());
  let shifted = input >> shift;
  if (input & mask) != T::zero() {
    shifted + (shifted & T::one())
  } else {
    shifted
  }
}

/// Right-shift an integer, but round towards positive infinity.
pub fn shr_ceil <T: Integer> (input: T, shift: u32)->T {
  let divisor = match T::one().checked_shl ( shift ) {Some (value) => value, None => return T::zero()};
  let mask = divisor.wrapping_sub (&T::one());
  (input >> shift) + if input & mask != T::zero() {T::one()} else {T::zero()}
}

/// Left-shift an integer, returning Some(input*(2^shift)) if it fits within the type, None otherwise.
pub fn overflow_checked_shl <T: Integer> (input: T, shift: u32)->Option <T> {
  if input == T::zero() {return Some (T::zero())}
  let maximum = match T::max_value().checked_shr (shift) {
    None => return None,
    Some (value) => value,
  };
  if input > maximum {return None}
  let minimum = T::min_value() >> shift;
  if input < minimum {return None}
  Some (input << shift)
}

/// Compute the arithmetic mean of two integers, rounded towards negative infinity. Never overflows.
pub fn mean_floor <T: Integer> (first: T, second: T)->T {
  (first >> 1u32) + (second >> 1u32) + (first & second & T::one())
}

/// Compute the arithmetic mean of two integers, rounded towards positive infinity. Never overflows.
pub fn mean_ceil <T: Integer> (first: T, second: T)->T {
  (first >> 1u32) + (second >> 1u32) + ((first | second) & T::one())
}

pub mod polynomial;

#[cfg (test)]
mod tests {
  use super::*;
  use num::{One, ToPrimitive, Integer};
  use num::bigint::BigInt;
  use num::rational::{Ratio};
  use std::cmp::Ordering;

  fn perfect_shr_nicely_rounded <T: Integer> (input: T, shift: u32)->BigInt where BigInt: From <T> {
    let perfect_result = Ratio::new (BigInt::from (input), BigInt::one() << shift as usize);
    let rounded_down = perfect_result.floor();
    let fraction = & perfect_result - & rounded_down;
    let rounded_down = rounded_down.to_integer();
    match fraction.cmp (& Ratio::new (BigInt::one(), BigInt::one() << 1)) {
      Ordering::Less => rounded_down, Ordering::Greater => rounded_down + BigInt::one() ,
      Ordering::Equal => & rounded_down + rounded_down.mod_floor (& (BigInt::one() << 1)),
    }
  }

  fn perfect_shr_round_to_even <T: Integer> (input: T, shift: u32)->BigInt where BigInt: From <T> {
    let perfect_result = Ratio::new (BigInt::from (input), BigInt::one() << shift as usize);
    let rounded_down = perfect_result.floor();
    if perfect_result == rounded_down {rounded_down.to_integer()}
    else {
      let rounded_down = rounded_down.to_integer();
      if &rounded_down >> 1 << 1 == rounded_down {
        rounded_down
      }
      else {rounded_down + 1}
    }
  }

  #[test]
  fn test_shr_nicely_rounded() {
    let inputs: Vec<(i64, u32, i64)> = vec![
      (0, 0, 0), (0, 5, 0), (1, 3, 0), (4, 3, 0), (5, 3, 1),
      (999, 1, 500), (998, 1, 499), (997, 1, 498)
    ];
    for (input, shift, result) in inputs {
      assert_eq!(shr_nicely_rounded (input, shift), result);
      assert_eq!(shr_nicely_rounded (-input, shift), -result);
    }
  }

  #[test]
  fn test_shr_round_to_even() {
    let inputs: Vec<(i64, u32, i64)> = vec![
      (0, 0, 0), (0, 5, 0), (1, 3, 0), (4, 3, 0), (5, 3, 0),
      (999, 1, 500), (998, 1, 499), (997, 1, 498)
    ];
    for (input, shift, result) in inputs {
      assert_eq!(shr_round_to_even (input, shift), result);
      assert_eq!(shr_round_to_even (-input, shift), -result);
    }
  }

  quickcheck! {
    fn quickcheck_shr_nicely_rounded_signed (input: i32, shift: u8)->bool {
      let result = shr_nicely_rounded (input, shift as u32);
      let perfect_result = perfect_shr_nicely_rounded (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_nicely_rounded_unsigned (input: u32, shift: u8)->bool {
      let result = shr_nicely_rounded (input, shift as u32);
      let perfect_result = perfect_shr_nicely_rounded (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_round_to_even_signed (input: i32, shift: u8)->bool {
      let result = shr_round_to_even (input, shift as u32);
      let perfect_result = perfect_shr_round_to_even (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_shr_round_to_even_unsigned (input: u32, shift: u8)->bool {
      let result = shr_round_to_even (input, shift as u32);
      let perfect_result = perfect_shr_round_to_even (input, shift as u32);
      println!( "{:?}", (result, & perfect_result.to_str_radix (10)));
      perfect_result == BigInt::from (result)
    }

    fn quickcheck_overflow_checked_shl (input: i32, shift: u8)->bool {
      let result = overflow_checked_shl (input, shift as u32);
      let perfect_result = BigInt::from (input) << shift as usize;
      result == perfect_result.to_i32()
    }
  }
}
<|endoftext|>


<commit_before>//! A shared ordered log.

use std::rc::Rc;
use std::cell::RefCell;
use std::time::{Instant, Duration};
use std::collections::VecDeque;

use ::{communication::Allocate, ExchangeData, PartialOrder};
use ::scheduling::Scheduler;
use worker::Worker;
use dataflow::channels::pact::Exchange;
use dataflow::operators::generic::operator::source;
use dataflow::operators::generic::operator::Operator;
use scheduling::activate::Activator;

// A Sequencer needs all operators firing with high frequency, because
// it uses the timer to gauge progress. If other workers cease
// advancing their own capabilities, although they might receive a
// record they may not actually tick forward their own source clocks,
// and no one will actually form the sequence.
//
// A CatchupActivator is an activator with an optional timestamp
// attached. This allows us to represent a special state, where after
// receiving an action from another worker, each of the other workers
// will keep scheduling its source operator, until its capability
// timestamp exceeds the greatest timestamp that the sink has
// received.
//
// This allows operators to go quiet again until a new requests shows
// up. The operators lose the ability to confirm that nothing is
// scheduled for a particular time (they could request this with a
// no-op event bearing a timestamp), but everyone still sees the same
// sequence.
struct CatchupActivator {
    pub catchup_until: Option<Duration>,
    activator: Activator,
}

impl CatchupActivator {
    pub fn activate(&self) {
        self.activator.activate();
    }
}

/// Orders elements inserted across all workers.
///
/// A Sequencer allows each worker to insert into a consistent ordered
/// sequence that is seen by all workers in the same order.
pub struct Sequencer<T> {
    activator: Rc<RefCell<Option<CatchupActivator>>>,
    send: Rc<RefCell<VecDeque<T>>>, // proposed items.
    recv: Rc<RefCell<VecDeque<T>>>, // sequenced items.
}

impl<T: ExchangeData> Sequencer<T> {

    /// Creates a new Sequencer.
    ///
    /// The `timer` instant is used to synchronize the workers, who use this
    /// elapsed time as their timestamp. Elements are ordered by this time,
    /// and cannot be made visible until all workers have reached the time.
    ///
    /// # Examples
    ///
    /// ```rust
    /// use std::time::{Instant, Duration};
    ///
    /// use timely::Configuration;
    /// use timely::synchronization::Sequencer;
    ///
    /// timely::execute(Configuration::Process(4), |worker| {
    ///     let timer = Instant::now();
    ///     let mut sequencer = Sequencer::new(worker, timer);
    ///
    ///     for round in 0 .. 10 {
    ///
    ///         // Sleep, and then send an announcement on wake-up.
    ///         std::thread::sleep(Duration::from_millis(1 + worker.index() as u64));
    ///         sequencer.push(format!("worker {:?}, round {:?}", worker.index(), round));
    ///
    ///         // Ensures the pushed string is sent.
    ///         worker.step();
    ///
    ///         // Read out received announcements.
    ///         while let Some(element) = sequencer.next() {
    ///             println!("{:?}:\tWorker {:?}:\t recv'd: {:?}", timer.elapsed(), worker.index(), element);
    ///         }
    ///     }
    /// }).expect("Timely computation did not complete correctly.");
    /// ```
    pub fn new<A: Allocate>(worker: &mut Worker<A>, timer: Instant) -> Self {
        Sequencer::preloaded(worker, timer, VecDeque::new())
    }

    /// Creates a new Sequencer preloaded with a queue of
    /// elements.
    pub fn preloaded<A: Allocate>(worker: &mut Worker<A>, timer: Instant, preload: VecDeque<T>) -> Self {

        let send: Rc<RefCell<VecDeque<T>>> = Rc::new(RefCell::new(VecDeque::new()));
        let recv = Rc::new(RefCell::new(preload));
        let send_weak = Rc::downgrade(&send);
        let recv_weak = Rc::downgrade(&recv);

        // The SequenceInput activator will be held by the sequencer,
        // by the operator itself, and by the sink operator. We can
        // only initialize the activator once we obtain the operator
        // address.
        let activator = Rc::new(RefCell::new(None));
        let activator_source = activator.clone();
        let activator_sink = activator.clone();

        // build a dataflow used to serialize and circulate commands
        worker.dataflow::<Duration,_,_>(move |dataflow| {

            let scope = dataflow.clone();
            let peers = dataflow.peers();

            let mut recvd = Vec::new();
            let mut vector = Vec::new();

            // monotonic counter to maintain per-worker total order.
            let mut counter = 0;

            // a source that attempts to pull from `recv` and produce commands for everyone
            source(dataflow, "SequenceInput", move |capability, info| {

                // intialize activator, now that we have the address
                activator_source
                    .borrow_mut()
                    .replace(CatchupActivator {
                        activator: scope.activator_for(&info.address[..]),
                        catchup_until: None,
                    });

                // so we can drop, if input queue vanishes.
                let mut capability = Some(capability);

                // closure broadcasts any commands it grabs.
                move |output| {

                    if let Some(send_queue) = send_weak.upgrade() {

                        // capability *should* still be non-None.
                        let capability = capability.as_mut().expect("Capability unavailable");

                        // downgrade capability to current time.
                        capability.downgrade(&timer.elapsed());

                        // drain and broadcast `send`.
                        let mut session = output.session(&capability);
                        let mut borrow = send_queue.borrow_mut();
                        for element in borrow.drain(..) {
                            for worker_index in 0 .. peers {
                                session.give((worker_index, counter, element.clone()));
                            }
                            counter += 1;
                        }

                        let mut activator_borrow = activator_source.borrow_mut();
                        let mut activator = activator_borrow.as_mut().unwrap();

                        if let Some(t) = activator.catchup_until {
                            if capability.time().less_than(&t) {
                                activator.activate();
                            } else {
                                activator.catchup_until = None;
                            }
                        }
                    } else {
                        capability = None;
                    }
                }
            })
            .sink(
                Exchange::new(|x: &(usize, usize, T)| x.0 as u64),
                "SequenceOutput",
                move |input| {

                    // grab each command and queue it up
                    input.for_each(|time, data| {
                        data.swap(&mut vector);
                        for (worker, counter, element) in vector.drain(..) {
                            recvd.push(((time.time().clone(), worker, counter), element));
                        }
                    });

                    recvd.sort_by(|x,y| x.0.cmp(&y.0));

                    if let Some(last) = recvd.last() {
                        let mut activator_borrow = activator_sink.borrow_mut();
                        let mut activator = activator_borrow.as_mut().unwrap();

                        activator.catchup_until = Some((last.0).0);
                        activator.activate();
                    }

                    // determine how many (which) elements to read from `recvd`.
                    let count = recvd.iter().filter(|&((ref time, _, _), _)| !input.frontier().less_equal(time)).count();
                    let iter = recvd.drain(..count);

                    if let Some(recv_queue) = recv_weak.upgrade() {
                        recv_queue.borrow_mut().extend(iter.map(|(_,elem)| elem));
                    }
                }
            );
        });

        Sequencer { activator, send, recv, }
    }

    /// Adds an element to the shared log.
    pub fn push(&mut self, element: T) {
        self.send.borrow_mut().push_back(element);
        self.activator.borrow_mut().as_mut().unwrap().activate();
    }
}

impl<T> Iterator for Sequencer<T> {
    type Item = T;
    fn next(&mut self) -> Option<T> {
        self.recv.borrow_mut().pop_front()
    }
}
<commit_msg>activate on drop<commit_after>//! A shared ordered log.

use std::rc::Rc;
use std::cell::RefCell;
use std::time::{Instant, Duration};
use std::collections::VecDeque;

use ::{communication::Allocate, ExchangeData, PartialOrder};
use ::scheduling::Scheduler;
use worker::Worker;
use dataflow::channels::pact::Exchange;
use dataflow::operators::generic::operator::source;
use dataflow::operators::generic::operator::Operator;
use scheduling::activate::Activator;

// A Sequencer needs all operators firing with high frequency, because
// it uses the timer to gauge progress. If other workers cease
// advancing their own capabilities, although they might receive a
// record they may not actually tick forward their own source clocks,
// and no one will actually form the sequence.
//
// A CatchupActivator is an activator with an optional timestamp
// attached. This allows us to represent a special state, where after
// receiving an action from another worker, each of the other workers
// will keep scheduling its source operator, until its capability
// timestamp exceeds the greatest timestamp that the sink has
// received.
//
// This allows operators to go quiet again until a new requests shows
// up. The operators lose the ability to confirm that nothing is
// scheduled for a particular time (they could request this with a
// no-op event bearing a timestamp), but everyone still sees the same
// sequence.
struct CatchupActivator {
    pub catchup_until: Option<Duration>,
    activator: Activator,
}

impl CatchupActivator {
    pub fn activate(&self) {
        self.activator.activate();
    }
}

/// Orders elements inserted across all workers.
///
/// A Sequencer allows each worker to insert into a consistent ordered
/// sequence that is seen by all workers in the same order.
pub struct Sequencer<T> {
    activator: Rc<RefCell<Option<CatchupActivator>>>,
    send: Rc<RefCell<VecDeque<T>>>, // proposed items.
    recv: Rc<RefCell<VecDeque<T>>>, // sequenced items.
}

impl<T: ExchangeData> Sequencer<T> {

    /// Creates a new Sequencer.
    ///
    /// The `timer` instant is used to synchronize the workers, who use this
    /// elapsed time as their timestamp. Elements are ordered by this time,
    /// and cannot be made visible until all workers have reached the time.
    ///
    /// # Examples
    ///
    /// ```rust
    /// use std::time::{Instant, Duration};
    ///
    /// use timely::Configuration;
    /// use timely::synchronization::Sequencer;
    ///
    /// timely::execute(Configuration::Process(4), |worker| {
    ///     let timer = Instant::now();
    ///     let mut sequencer = Sequencer::new(worker, timer);
    ///
    ///     for round in 0 .. 10 {
    ///
    ///         // Sleep, and then send an announcement on wake-up.
    ///         std::thread::sleep(Duration::from_millis(1 + worker.index() as u64));
    ///         sequencer.push(format!("worker {:?}, round {:?}", worker.index(), round));
    ///
    ///         // Ensures the pushed string is sent.
    ///         worker.step();
    ///
    ///         // Read out received announcements.
    ///         while let Some(element) = sequencer.next() {
    ///             println!("{:?}:\tWorker {:?}:\t recv'd: {:?}", timer.elapsed(), worker.index(), element);
    ///         }
    ///     }
    /// }).expect("Timely computation did not complete correctly.");
    /// ```
    pub fn new<A: Allocate>(worker: &mut Worker<A>, timer: Instant) -> Self {
        Sequencer::preloaded(worker, timer, VecDeque::new())
    }

    /// Creates a new Sequencer preloaded with a queue of
    /// elements.
    pub fn preloaded<A: Allocate>(worker: &mut Worker<A>, timer: Instant, preload: VecDeque<T>) -> Self {

        let send: Rc<RefCell<VecDeque<T>>> = Rc::new(RefCell::new(VecDeque::new()));
        let recv = Rc::new(RefCell::new(preload));
        let send_weak = Rc::downgrade(&send);
        let recv_weak = Rc::downgrade(&recv);

        // The SequenceInput activator will be held by the sequencer,
        // by the operator itself, and by the sink operator. We can
        // only initialize the activator once we obtain the operator
        // address.
        let activator = Rc::new(RefCell::new(None));
        let activator_source = activator.clone();
        let activator_sink = activator.clone();

        // build a dataflow used to serialize and circulate commands
        worker.dataflow::<Duration,_,_>(move |dataflow| {

            let scope = dataflow.clone();
            let peers = dataflow.peers();

            let mut recvd = Vec::new();
            let mut vector = Vec::new();

            // monotonic counter to maintain per-worker total order.
            let mut counter = 0;

            // a source that attempts to pull from `recv` and produce commands for everyone
            source(dataflow, "SequenceInput", move |capability, info| {

                // intialize activator, now that we have the address
                activator_source
                    .borrow_mut()
                    .replace(CatchupActivator {
                        activator: scope.activator_for(&info.address[..]),
                        catchup_until: None,
                    });

                // so we can drop, if input queue vanishes.
                let mut capability = Some(capability);

                // closure broadcasts any commands it grabs.
                move |output| {

                    if let Some(send_queue) = send_weak.upgrade() {

                        // capability *should* still be non-None.
                        let capability = capability.as_mut().expect("Capability unavailable");

                        // downgrade capability to current time.
                        capability.downgrade(&timer.elapsed());

                        // drain and broadcast `send`.
                        let mut session = output.session(&capability);
                        let mut borrow = send_queue.borrow_mut();
                        for element in borrow.drain(..) {
                            for worker_index in 0 .. peers {
                                session.give((worker_index, counter, element.clone()));
                            }
                            counter += 1;
                        }

                        let mut activator_borrow = activator_source.borrow_mut();
                        let mut activator = activator_borrow.as_mut().unwrap();

                        if let Some(t) = activator.catchup_until {
                            if capability.time().less_than(&t) {
                                activator.activate();
                            } else {
                                activator.catchup_until = None;
                            }
                        }
                    } else {
                        println!("Shutting down sequencer");
                        capability = None;
                    }
                }
            })
            .sink(
                Exchange::new(|x: &(usize, usize, T)| x.0 as u64),
                "SequenceOutput",
                move |input| {

                    // grab each command and queue it up
                    input.for_each(|time, data| {
                        data.swap(&mut vector);
                        for (worker, counter, element) in vector.drain(..) {
                            recvd.push(((time.time().clone(), worker, counter), element));
                        }
                    });

                    recvd.sort_by(|x,y| x.0.cmp(&y.0));

                    if let Some(last) = recvd.last() {
                        let mut activator_borrow = activator_sink.borrow_mut();
                        let mut activator = activator_borrow.as_mut().unwrap();

                        activator.catchup_until = Some((last.0).0);
                        activator.activate();
                    }

                    // determine how many (which) elements to read from `recvd`.
                    let count = recvd.iter().filter(|&((ref time, _, _), _)| !input.frontier().less_equal(time)).count();
                    let iter = recvd.drain(..count);

                    if let Some(recv_queue) = recv_weak.upgrade() {
                        recv_queue.borrow_mut().extend(iter.map(|(_,elem)| elem));
                    }
                }
            );
        });

        Sequencer { activator, send, recv, }
    }

    /// Adds an element to the shared log.
    pub fn push(&mut self, element: T) {
        self.send.borrow_mut().push_back(element);
        self.activator.borrow_mut().as_mut().unwrap().activate();
    }
}

impl<T> Iterator for Sequencer<T> {
    type Item = T;
    fn next(&mut self) -> Option<T> {
        self.recv.borrow_mut().pop_front()
    }
}

// We should activate on drop, as this will cause the source to drop its capability.
impl<T> Drop for Sequencer<T> {
    fn drop(&mut self) {
        self.activator
            .borrow()
            .as_ref()
            .expect("Sequencer.activator unavailable")
            .activate()
    }
}<|endoftext|>