<commit_before>use html5ever::{QualName, parse_fragment};
use html5ever::rcdom::{NodeData, RcDom, Handle};
use html5ever::tendril::TendrilSink;

error_chain! {
    errors {
        HtmlParse(errors: Vec<String>) {
            description("There was a problem parsing the HTML")
        }
    }
}

pub fn convert(html: &str) -> Result<String> {
    let mut converter = HtmlVisitor::new();

    let context = QualName::new(None, ns!(html), local_name!("body"));
    let dom = parse_fragment(RcDom::default(), Default::default(), context, vec![]).one(html);

    converter.visit(dom.document);

    if !dom.errors.is_empty() {
        bail!(ErrorKind::HtmlParse(dom.errors.into_iter().map(|e| e.into_owned()).collect()));
    }

    Ok(converter.markdown())
}

struct HtmlVisitor {
    markdown: String,
}

impl HtmlVisitor {
    fn new() -> HtmlVisitor {
        HtmlVisitor {
            markdown: String::new(),
        }
    }

    fn markdown(self) -> String {
        self.markdown
    }

    fn visit(&mut self, node: Handle) {
        match node.data {
            NodeData::Document => self.visit_children(node.clone()),
            NodeData::Text { .. } => self.visit_text(node.clone()),
            NodeData::Element { ref name, .. } => {
                match name.local {
                    local_name!("a") => self.visit_link(node.clone()),
                    local_name!("p") => self.visit_paragraph(node.clone()),
                    local_name!("code") => self.visit_code(node.clone()),
                    local_name!("i") => self.visit_italic(node.clone()),
                    _ => self.visit_children(node.clone()),
                }
            }
            NodeData::Doctype { .. } |
            NodeData::Comment { .. } |
            NodeData::ProcessingInstruction { .. } => {}
        }
    }

    fn visit_children(&mut self, node: Handle) {
        for child in node.children.borrow().iter() {
            self.visit(child.clone());
        }
    }

    fn visit_link(&mut self, node: Handle) {
        if let NodeData::Element { ref attrs, .. } = node.data {
            for attr in attrs.borrow().iter() {
                if &attr.name.local == "href" {
                    self.markdown.push_str(&*attr.value);
                }
            }
        }
    }

    fn visit_text(&mut self, node: Handle) {
        if let NodeData::Text { ref contents } = node.data {
            self.markdown.push_str(&contents.borrow());
        }
    }

    fn visit_paragraph(&mut self, node: Handle) {
        self.markdown.push_str("\n\n");

        self.visit_children(node);
    }

    fn visit_code(&mut self, node: Handle) {
        self.markdown.push_str("```\n");

        self.visit_children(node);

        self.markdown.push_str("```");
    }

    fn visit_italic(&mut self, node: Handle) {
        self.markdown.push_str("*");

        self.visit_children(node);

        self.markdown.push_str("*");
    }
}

#[test]
fn test_html_to_markdown() {
    use std::path::Path;
    use std::fs::File;
    use std::io::Read;

    let fixtures =
        Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/hacker_news/html_to_markdown/");

    let mut input_file = File::open(fixtures.join("input.html")).expect("Couldn't open fixture");
    let mut input = String::new();
    input_file
        .read_to_string(&mut input)
        .expect("Couldn't read fixture");

    let parsed = convert(&input).expect("Couldn't convert HTML to Markdown");

    let mut expected_file =
        File::open(fixtures.join("expected.markdown")).expect("Couldn't open fixture");
    let mut expected = String::new();
    expected_file
        .read_to_string(&mut expected)
        .expect("Couldn't read fixture");

    assert_eq!(expected, parsed);
}
<commit_msg>use conventional naming<commit_after>use html5ever::{QualName, parse_fragment};
use html5ever::rcdom::{NodeData, RcDom, Handle};
use html5ever::tendril::TendrilSink;

error_chain! {
    errors {
        HtmlParse(errors: Vec<String>) {
            description("There was a problem parsing the HTML")
        }
    }
}

pub fn convert(html: &str) -> Result<String> {
    let mut converter = HtmlVisitor::new();

    let context = QualName::new(None, ns!(html), local_name!("body"));
    let dom = parse_fragment(RcDom::default(), Default::default(), context, vec![]).one(html);

    converter.visit(dom.document);

    if !dom.errors.is_empty() {
        bail!(ErrorKind::HtmlParse(dom.errors.into_iter().map(|e| e.into_owned()).collect()));
    }

    Ok(converter.into_markdown())
}

struct HtmlVisitor {
    markdown: String,
}

impl HtmlVisitor {
    fn new() -> HtmlVisitor {
        HtmlVisitor {
            markdown: String::new(),
        }
    }

    /// Return the underlying converted Markdown.
    fn into_markdown(self) -> String {
        self.markdown
    }

    fn visit(&mut self, node: Handle) {
        match node.data {
            NodeData::Document => self.visit_children(node.clone()),
            NodeData::Text { .. } => self.visit_text(node.clone()),
            NodeData::Element { ref name, .. } => {
                match name.local {
                    local_name!("a") => self.visit_link(node.clone()),
                    local_name!("p") => self.visit_paragraph(node.clone()),
                    local_name!("code") => self.visit_code(node.clone()),
                    local_name!("i") => self.visit_italic(node.clone()),
                    _ => self.visit_children(node.clone()),
                }
            }
            NodeData::Doctype { .. } |
            NodeData::Comment { .. } |
            NodeData::ProcessingInstruction { .. } => {}
        }
    }

    fn visit_children(&mut self, node: Handle) {
        for child in node.children.borrow().iter() {
            self.visit(child.clone());
        }
    }

    fn visit_link(&mut self, node: Handle) {
        if let NodeData::Element { ref attrs, .. } = node.data {
            for attr in attrs.borrow().iter() {
                if &attr.name.local == "href" {
                    self.markdown.push_str(&*attr.value);
                }
            }
        }
    }

    fn visit_text(&mut self, node: Handle) {
        if let NodeData::Text { ref contents } = node.data {
            self.markdown.push_str(&contents.borrow());
        }
    }

    fn visit_paragraph(&mut self, node: Handle) {
        self.markdown.push_str("\n\n");

        self.visit_children(node);
    }

    fn visit_code(&mut self, node: Handle) {
        self.markdown.push_str("```\n");

        self.visit_children(node);

        self.markdown.push_str("```");
    }

    fn visit_italic(&mut self, node: Handle) {
        self.markdown.push_str("*");

        self.visit_children(node);

        self.markdown.push_str("*");
    }
}

#[test]
fn test_html_to_markdown() {
    use std::path::Path;
    use std::fs::File;
    use std::io::Read;

    let fixtures =
        Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/hacker_news/html_to_markdown/");

    let mut input_file = File::open(fixtures.join("input.html")).expect("Couldn't open fixture");
    let mut input = String::new();
    input_file
        .read_to_string(&mut input)
        .expect("Couldn't read fixture");

    let parsed = convert(&input).expect("Couldn't convert HTML to Markdown");

    let mut expected_file =
        File::open(fixtures.join("expected.markdown")).expect("Couldn't open fixture");
    let mut expected = String::new();
    expected_file
        .read_to_string(&mut expected)
        .expect("Couldn't read fixture");

    assert_eq!(expected, parsed);
}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
extern crate clap;
#[macro_use] extern crate semver;
extern crate toml;
extern crate url;
#[macro_use] extern crate version;

extern crate libimaglink;
extern crate libimagrt;
extern crate libimagstore;
extern crate libimagutil;

use std::process::exit;
use std::ops::Deref;
use std::error::Error;

use libimagrt::runtime::Runtime;
use libimagstore::error::StoreError;
use libimagstore::store::Entry;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Store;
use libimagutil::trace::trace_error;
use libimaglink::external::ExternalLinker;
use clap::ArgMatches;
use url::Url;

mod ui;

use ui::build_ui;

fn main() {
    let name = "imag-link";
    let version = &version!()[..];
    let about = "Link entries";
    let ui = build_ui(Runtime::get_default_cli_builder(name, version, about));
    let rt = {
        let rt = Runtime::new(ui);
        if rt.is_ok() {
            rt.unwrap()
        } else {
            println!("Could not set up Runtime");
            println!("{:?}", rt.err().unwrap());
            exit(1);
        }
    };

    debug!("Hello. Logging was just enabled");
    debug!("I already set up the Runtime object and build the commandline interface parser.");
    debug!("Lets get rollin' ...");

    rt.cli()
        .subcommand_name()
        .map(|name| {
            match name {
                "internal" => handle_internal_linking(&rt),
                "external" => handle_external_linking(&rt),
                _ => {
                    warn!("No commandline call");
                    exit(1);
                },
            }
        });
}

fn handle_internal_linking(rt: &Runtime) {
    use libimaglink::internal::InternalLinker;
    use libimagutil::trace::trace_error;

    debug!("Handle internal linking call");
    let cmd = rt.cli().subcommand_matches("internal").unwrap();

    if cmd.is_present("list") {
        debug!("List...");
        for entry in cmd.value_of("list").unwrap().split(",") {
            debug!("Listing for '{}'", entry);
            match get_entry_by_name(rt, entry) {
                Ok(e) => {
                    e.get_internal_links()
                        .map(|links| {
                            let mut i = 0;
                            for link in links.iter().map(|l| l.to_str()).filter_map(|x| x) {
                                println!("{: <3}: {}", i, link);
                                i += 1;
                            }
                        });
                },

                Err(e) => {
                    trace_error(&e);
                    break;
                },
            }
        }
        debug!("Listing ready!");
    } else {
        let mut from = {
            let mut from = get_from_entry(&rt);
            if from.is_none() {
                warn!("No 'from' entry");
                exit(1);
            }
            from.unwrap()
        };
        debug!("Link from = {:?}", from.deref());

        let mut to = {
            let mut to = get_to_entries(&rt);
            if to.is_none() {
                warn!("No 'to' entry");
                exit(1);
            }
            to.unwrap()
        };
        debug!("Link to = {:?}", to.iter().map(|f| f.deref()).collect::<Vec<&Entry>>());

        match cmd.subcommand_name() {
            Some("add") => {
                for mut to_entry in to {
                    if let Err(e) = to_entry.add_internal_link(&mut from) {
                        trace_error(&e);
                        exit(1);
                    }
                }
            },

            Some("remove") => {
                for mut to_entry in to {
                    if let Err(e) = to_entry.remove_internal_link(&mut from) {
                        trace_error(&e);
                        exit(1);
                    }
                }
            },

            _ => unreachable!(),
        };
    }
}

fn get_from_entry<'a>(rt: &'a Runtime) -> Option<FileLockEntry<'a>> {
    rt.cli()
        .subcommand_matches("internal")
        .unwrap() // safe, we know there is an "internal" subcommand"
        .subcommand_matches("add")
        .unwrap() // safe, we know there is an "add" subcommand
        .value_of("from")
        .and_then(|from_name| {
            match get_entry_by_name(rt, from_name) {
                Err(e) => {
                    debug!("We couldn't get the entry from name: '{:?}'", from_name);
                    trace_error(&e); None
                },
                Ok(e) => Some(e),
            }

        })
}

fn get_to_entries<'a>(rt: &'a Runtime) -> Option<Vec<FileLockEntry<'a>>> {
    rt.cli()
        .subcommand_matches("internal")
        .unwrap() // safe, we know there is an "internal" subcommand"
        .subcommand_matches("add")
        .unwrap() // safe, we know there is an "add" subcommand
        .values_of("to")
        .map(|values| {
            let mut v = vec![];
            for entry in values.map(|v| get_entry_by_name(rt, v)) {
                match entry {
                    Err(e) => trace_error(&e),
                    Ok(e) => v.push(e),
                }
            }
            v
        })
}

fn get_entry_by_name<'a>(rt: &'a Runtime, name: &str) -> Result<FileLockEntry<'a>, StoreError> {
    use libimagstore::storeid::build_entry_path;
    build_entry_path(rt.store(), name)
        .and_then(|path| rt.store().retrieve(path))
}

fn handle_external_linking(rt: &Runtime) {
    use libimagutil::trace::trace_error;

    let scmd       = rt.cli().subcommand_matches("external").unwrap();
    let entry_name = scmd.value_of("id").unwrap(); // enforced by clap
    let entry      = get_entry_by_name(rt, entry_name);
    if entry.is_err() {
        trace_error(&entry.err().unwrap());
        exit(1);
    }
    let mut entry = entry.unwrap();

    if scmd.is_present("add") {
        debug!("Adding link to entry!");
        add_link_to_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("remove") {
        debug!("Removing link from entry!");
        remove_link_from_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("set") {
        debug!("Setting links in entry!");
        set_links_for_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("list") {
        debug!("Listing links in entry!");
        list_links_for_entry(rt.store(), &mut entry);
        return;
    }

    panic!("Clap failed to enforce one of 'add', 'remove', 'set' or 'list'");
}

fn add_link_to_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let link = matches.value_of("add").unwrap();

    let link = Url::parse(link);
    if link.is_err() {
        debug!("URL parsing error...");
        trace_error(&link.err().unwrap());
        debug!("Exiting");
        exit(1);
    }
    let link = link.unwrap();

    if let Err(e) = entry.add_external_link(store, link) {
        debug!("Error while adding external link...");
        trace_error(&e);
    } else {
        debug!("Everything worked well");
        info!("Ok");
    }
}

fn remove_link_from_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let link = matches.value_of("remove").unwrap();

    let link = Url::parse(link);
    if link.is_err() {
        trace_error(&link.err().unwrap());
        exit(1);
    }
    let link = link.unwrap();

    if let Err(e) = entry.remove_external_link(store, link) {
        trace_error(&e);
    } else {
        info!("Ok");
    }
}

fn set_links_for_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let links = matches
        .value_of("links")
        .map(String::from)
        .unwrap()
        .split(",")
        .map(|uri| {
            match Url::parse(uri) {
                Err(e) => {
                    warn!("Could not parse '{}' as URL, ignoring", uri);
                    trace_error(&e);
                    None
                },
                Ok(u) => Some(u),
            }
        })
        .filter_map(|x| x)
        .collect();

    if let Err(e) = entry.set_external_links(store, links) {
        trace_error(&e);
    } else {
        info!("Ok");
    }
}

fn list_links_for_entry(store: &Store, entry: &mut FileLockEntry) {
    let res = entry.get_external_links(store)
        .and_then(|links| {
            let mut i = 0;
            for link in links {
                println!("{: <3}: {}", i, link);
                i += 1;
            }
            Ok(())
        });

    match res {
        Err(e) => {
            trace_error(&e);
        },
        Ok(_) => {
            info!("Ok");
        },
    }
}

<commit_msg>Remove init-logging from imag-link<commit_after>#[macro_use] extern crate log;
extern crate clap;
#[macro_use] extern crate semver;
extern crate toml;
extern crate url;
#[macro_use] extern crate version;

extern crate libimaglink;
extern crate libimagrt;
extern crate libimagstore;
extern crate libimagutil;

use std::process::exit;
use std::ops::Deref;
use std::error::Error;

use libimagrt::runtime::Runtime;
use libimagstore::error::StoreError;
use libimagstore::store::Entry;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Store;
use libimagutil::trace::trace_error;
use libimaglink::external::ExternalLinker;
use clap::ArgMatches;
use url::Url;

mod ui;

use ui::build_ui;

fn main() {
    let name = "imag-link";
    let version = &version!()[..];
    let about = "Link entries";
    let ui = build_ui(Runtime::get_default_cli_builder(name, version, about));
    let rt = {
        let rt = Runtime::new(ui);
        if rt.is_ok() {
            rt.unwrap()
        } else {
            println!("Could not set up Runtime");
            println!("{:?}", rt.err().unwrap());
            exit(1);
        }
    };

    rt.cli()
        .subcommand_name()
        .map(|name| {
            match name {
                "internal" => handle_internal_linking(&rt),
                "external" => handle_external_linking(&rt),
                _ => {
                    warn!("No commandline call");
                    exit(1);
                },
            }
        });
}

fn handle_internal_linking(rt: &Runtime) {
    use libimaglink::internal::InternalLinker;
    use libimagutil::trace::trace_error;

    debug!("Handle internal linking call");
    let cmd = rt.cli().subcommand_matches("internal").unwrap();

    if cmd.is_present("list") {
        debug!("List...");
        for entry in cmd.value_of("list").unwrap().split(",") {
            debug!("Listing for '{}'", entry);
            match get_entry_by_name(rt, entry) {
                Ok(e) => {
                    e.get_internal_links()
                        .map(|links| {
                            let mut i = 0;
                            for link in links.iter().map(|l| l.to_str()).filter_map(|x| x) {
                                println!("{: <3}: {}", i, link);
                                i += 1;
                            }
                        });
                },

                Err(e) => {
                    trace_error(&e);
                    break;
                },
            }
        }
        debug!("Listing ready!");
    } else {
        let mut from = {
            let mut from = get_from_entry(&rt);
            if from.is_none() {
                warn!("No 'from' entry");
                exit(1);
            }
            from.unwrap()
        };
        debug!("Link from = {:?}", from.deref());

        let mut to = {
            let mut to = get_to_entries(&rt);
            if to.is_none() {
                warn!("No 'to' entry");
                exit(1);
            }
            to.unwrap()
        };
        debug!("Link to = {:?}", to.iter().map(|f| f.deref()).collect::<Vec<&Entry>>());

        match cmd.subcommand_name() {
            Some("add") => {
                for mut to_entry in to {
                    if let Err(e) = to_entry.add_internal_link(&mut from) {
                        trace_error(&e);
                        exit(1);
                    }
                }
            },

            Some("remove") => {
                for mut to_entry in to {
                    if let Err(e) = to_entry.remove_internal_link(&mut from) {
                        trace_error(&e);
                        exit(1);
                    }
                }
            },

            _ => unreachable!(),
        };
    }
}

fn get_from_entry<'a>(rt: &'a Runtime) -> Option<FileLockEntry<'a>> {
    rt.cli()
        .subcommand_matches("internal")
        .unwrap() // safe, we know there is an "internal" subcommand"
        .subcommand_matches("add")
        .unwrap() // safe, we know there is an "add" subcommand
        .value_of("from")
        .and_then(|from_name| {
            match get_entry_by_name(rt, from_name) {
                Err(e) => {
                    debug!("We couldn't get the entry from name: '{:?}'", from_name);
                    trace_error(&e); None
                },
                Ok(e) => Some(e),
            }

        })
}

fn get_to_entries<'a>(rt: &'a Runtime) -> Option<Vec<FileLockEntry<'a>>> {
    rt.cli()
        .subcommand_matches("internal")
        .unwrap() // safe, we know there is an "internal" subcommand"
        .subcommand_matches("add")
        .unwrap() // safe, we know there is an "add" subcommand
        .values_of("to")
        .map(|values| {
            let mut v = vec![];
            for entry in values.map(|v| get_entry_by_name(rt, v)) {
                match entry {
                    Err(e) => trace_error(&e),
                    Ok(e) => v.push(e),
                }
            }
            v
        })
}

fn get_entry_by_name<'a>(rt: &'a Runtime, name: &str) -> Result<FileLockEntry<'a>, StoreError> {
    use libimagstore::storeid::build_entry_path;
    build_entry_path(rt.store(), name)
        .and_then(|path| rt.store().retrieve(path))
}

fn handle_external_linking(rt: &Runtime) {
    use libimagutil::trace::trace_error;

    let scmd       = rt.cli().subcommand_matches("external").unwrap();
    let entry_name = scmd.value_of("id").unwrap(); // enforced by clap
    let entry      = get_entry_by_name(rt, entry_name);
    if entry.is_err() {
        trace_error(&entry.err().unwrap());
        exit(1);
    }
    let mut entry = entry.unwrap();

    if scmd.is_present("add") {
        debug!("Adding link to entry!");
        add_link_to_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("remove") {
        debug!("Removing link from entry!");
        remove_link_from_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("set") {
        debug!("Setting links in entry!");
        set_links_for_entry(rt.store(), scmd, &mut entry);
        return;
    }

    if scmd.is_present("list") {
        debug!("Listing links in entry!");
        list_links_for_entry(rt.store(), &mut entry);
        return;
    }

    panic!("Clap failed to enforce one of 'add', 'remove', 'set' or 'list'");
}

fn add_link_to_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let link = matches.value_of("add").unwrap();

    let link = Url::parse(link);
    if link.is_err() {
        debug!("URL parsing error...");
        trace_error(&link.err().unwrap());
        debug!("Exiting");
        exit(1);
    }
    let link = link.unwrap();

    if let Err(e) = entry.add_external_link(store, link) {
        debug!("Error while adding external link...");
        trace_error(&e);
    } else {
        debug!("Everything worked well");
        info!("Ok");
    }
}

fn remove_link_from_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let link = matches.value_of("remove").unwrap();

    let link = Url::parse(link);
    if link.is_err() {
        trace_error(&link.err().unwrap());
        exit(1);
    }
    let link = link.unwrap();

    if let Err(e) = entry.remove_external_link(store, link) {
        trace_error(&e);
    } else {
        info!("Ok");
    }
}

fn set_links_for_entry(store: &Store, matches: &ArgMatches, entry: &mut FileLockEntry) {
    let links = matches
        .value_of("links")
        .map(String::from)
        .unwrap()
        .split(",")
        .map(|uri| {
            match Url::parse(uri) {
                Err(e) => {
                    warn!("Could not parse '{}' as URL, ignoring", uri);
                    trace_error(&e);
                    None
                },
                Ok(u) => Some(u),
            }
        })
        .filter_map(|x| x)
        .collect();

    if let Err(e) = entry.set_external_links(store, links) {
        trace_error(&e);
    } else {
        info!("Ok");
    }
}

fn list_links_for_entry(store: &Store, entry: &mut FileLockEntry) {
    let res = entry.get_external_links(store)
        .and_then(|links| {
            let mut i = 0;
            for link in links {
                println!("{: <3}: {}", i, link);
                i += 1;
            }
            Ok(())
        });

    match res {
        Err(e) => {
            trace_error(&e);
        },
        Ok(_) => {
            info!("Ok");
        },
    }
}

<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use cxx::{memory::UniquePtrTarget, UniquePtr};
use moveit::{CopyNew, New};
use std::{marker::PhantomPinned, mem::MaybeUninit, pin::Pin};

/// A trait representing a parameter to a C++ function which is received
/// by value.
///
/// Rust has the concept of receiving parameters by _move_ or by _reference_.
/// C++ has the concept of receiving a parameter by 'value', which means
/// the parameter gets copied.
///
/// To make it easy to pass such parameters from Rust, this trait exists.
/// It is implemented both for references `&T` and for `UniquePtr<T>`,
/// subject to the presence or absence of suitable copy and move constructors.
/// This allows you to pass in parameters by copy (as is ergonomic and normal
/// in C++) retaining the original parameter; or by move semantics thus
/// destroying the object you're passing in. Simply use a reference if you want
/// copy semantics, or the item itself if you want move semantics.
///
/// It is not recommended that you implement this trait, nor that you directly
/// use its methods, which are for use by `autocxx` generated code only.
///
/// # Use of `moveit` traits
///
/// Most of the implementations of this trait require the type to implement
/// [`moveit::CopyNew`], which is simply the `autocxx`/`moveit` way of saying that
/// the type has a copy constructor in C++.
///
/// # Performance
///
/// At present, some additional copying occurs for all implementations of
/// this trait other than that for [`cxx::UniquePtr`]. In the future it's
/// hoped that the implementation for `&T where T: CopyNew` can also avoid
/// this extra copying.
///
/// # Panics
///
/// The implementations of this trait which take a [`cxx::UniquePtr`] will
/// panic if the pointer is NULL.
pub trait ValueParam<T> {
    /// Any stack storage required. If, as part of passing to C++,
    /// we need to store a temporary copy of the value, this will be `T`,
    /// otherwise `()`.
    #[doc(hidden)]
    type StackStorage;
    /// Whether this `ValueParam` requires temporary Rust-side storage of
    /// an extra copy of this value.
    #[doc(hidden)]
    fn needs_stack_space(&self) -> bool;
    /// Populate the stack storage given as a parameter. Only called if you
    /// return `true` from `needs_stack_space`.
    #[doc(hidden)]
    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>);
    /// Return a pointer to the storage.
    /// Only called if `needs_stack_space` returns `false`, otherwise the pointer
    /// to the stack space will be used.
    /// Note that this returns a _mutable_ pointer. This is a big deal. That's
    /// because, on the C++ side, we'll call `std::move(*ptr)` on this pointer.
    /// This is unlikely to be semantically acceptable long-term, but currently
    /// makes it 'quicker' when a [`cxx::UniquePtr`] is used as a value parameter.
    #[doc(hidden)]
    fn get_ptr(&mut self) -> *mut T;
}

impl<T> ValueParam<T> for &T
where
    T: CopyNew,
{
    type StackStorage = T;

    fn needs_stack_space(&self) -> bool {
        true
    }

    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>) {
        unsafe { crate::moveit::new::copy(*self).new(this) }
    }

    fn get_ptr(&mut self) -> *mut T {
        unreachable!()
    }
}

impl<T> ValueParam<T> for UniquePtr<T>
where
    T: UniquePtrTarget,
{
    type StackStorage = ();

    fn needs_stack_space(&self) -> bool {
        false
    }

    fn populate_stack_space(&self, _: Pin<&mut MaybeUninit<Self::StackStorage>>) {}

    fn get_ptr(&mut self) -> *mut T {
        (unsafe {
            Pin::into_inner_unchecked(
                self.as_mut()
                    .expect("Passed a NULL UniquePtr as a C++ value parameter"),
            )
        }) as *mut T
    }
}

impl<T> ValueParam<T> for &UniquePtr<T>
where
    T: UniquePtrTarget + CopyNew,
{
    type StackStorage = T;

    fn needs_stack_space(&self) -> bool {
        true
    }

    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>) {
        // Invoke a copy constructor on the Rust side, because we'll use std::move on the C++
        // side.
        unsafe {
            crate::moveit::new::copy(
                self.as_ref()
                    .expect("Passed a NULL &UniquePtr as a C++ value parameter"),
            )
            .new(this)
        }
    }

    fn get_ptr(&mut self) -> *mut T {
        unreachable!()
    }
}

/// Implementation detail for how we pass value parameters into C++.
/// This type is instantiated by auto-generated autocxx code each time we
/// need to pass a value parameter into C++, and will take responsibility
/// for extracting that value parameter from the [`ValueParam`] and doing
/// any later cleanup.
#[doc(hidden)]
pub struct ValueParamHandler<T, VP: ValueParam<T>> {
    param: VP,
    space: Option<MaybeUninit<VP::StackStorage>>,
    _pinned: PhantomPinned,
}

impl<T, VP: ValueParam<T>> ValueParamHandler<T, VP> {
    /// Create a new storage space for something that's about to be passed
    /// by value to C++. Depending on the `ValueParam` type passed in,
    /// this may be largely a no-op or it may involve storing a whole
    /// extra copy of the type.
    ///
    /// # Safety
    ///
    /// Callers must guarantee that this type will not move
    /// in memory.
    pub unsafe fn new(param: VP) -> Self {
        let mut this = Self {
            param,
            space: None,
            _pinned: PhantomPinned,
        };
        if this.param.needs_stack_space() {
            this.space = Some(MaybeUninit::uninit());
            this.param
                .populate_stack_space(Pin::new_unchecked(this.space.as_mut().unwrap()));
        }
        this
    }

    /// Return a pointer to the underlying value which can be passed to C++.
    /// Per the unsafety contract of `new`, the object must not have moved
    /// since it was created.
    pub fn get_ptr(&mut self) -> *mut T {
        if let Some(ref mut space) = self.space {
            let ptr =
                unsafe { space.assume_init_mut() } as *mut <VP as ValueParam<T>>::StackStorage;
            unsafe { std::mem::transmute(ptr) }
        } else {
            self.param.get_ptr()
        }
    }
}

impl<T, VP: ValueParam<T>> Drop for ValueParamHandler<T, VP> {
    fn drop(&mut self) {
        if let Some(space) = self.space.take() {
            unsafe { space.assume_init() };
        }
    }
}
<commit_msg>Revert "Doc fix"<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use cxx::{memory::UniquePtrTarget, UniquePtr};
use moveit::{CopyNew, New};
use std::{marker::PhantomPinned, mem::MaybeUninit, pin::Pin};

/// A trait representing a parameter to a C++ function which is received
/// by value.
///
/// Rust has the concept of receiving parameters by _move_ or by _reference_.
/// C++ has the concept of receiving a parameter by 'value', which means
/// the parameter gets copied.
///
/// To make it easy to pass such parameters from Rust, this trait exists.
/// It is implemented both for references `&T` and for `UniquePtr<T>`,
/// subject to the presence or absence of suitable copy and move constructors.
/// This allows you to pass in parameters by copy (as is ergonomic and normal
/// in C++) retaining the original parameter; or by move semantics thus
/// destroying the object you're passing in. Simply use a reference if you want
/// copy semantics, or the item itself if you want move semantics.
///
/// It is not recommended that you implement this trait, nor that you directly
/// use its methods, which are for use by `autocxx` generated code only.
///
/// # Use of `moveit` traits
///
/// Most of the implementations of this trait require the type to implement
/// [`CopyNew`], which is simply the `autocxx`/`moveit` way of saying that
/// the type has a copy constructor in C++.
///
/// # Performance
///
/// At present, some additional copying occurs for all implementations of
/// this trait other than that for [`cxx::UniquePtr`]. In the future it's
/// hoped that the implementation for `&T where T: CopyNew` can also avoid
/// this extra copying.
///
/// # Panics
///
/// The implementations of this trait which take a [`cxx::UniquePtr`] will
/// panic if the pointer is NULL.
pub trait ValueParam<T> {
    /// Any stack storage required. If, as part of passing to C++,
    /// we need to store a temporary copy of the value, this will be `T`,
    /// otherwise `()`.
    #[doc(hidden)]
    type StackStorage;
    /// Whether this `ValueParam` requires temporary Rust-side storage of
    /// an extra copy of this value.
    #[doc(hidden)]
    fn needs_stack_space(&self) -> bool;
    /// Populate the stack storage given as a parameter. Only called if you
    /// return `true` from `needs_stack_space`.
    #[doc(hidden)]
    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>);
    /// Return a pointer to the storage.
    /// Only called if `needs_stack_space` returns `false`, otherwise the pointer
    /// to the stack space will be used.
    /// Note that this returns a _mutable_ pointer. This is a big deal. That's
    /// because, on the C++ side, we'll call `std::move(*ptr)` on this pointer.
    /// This is unlikely to be semantically acceptable long-term, but currently
    /// makes it 'quicker' when a [`cxx::UniquePtr`] is used as a value parameter.
    #[doc(hidden)]
    fn get_ptr(&mut self) -> *mut T;
}

impl<T> ValueParam<T> for &T
where
    T: CopyNew,
{
    type StackStorage = T;

    fn needs_stack_space(&self) -> bool {
        true
    }

    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>) {
        unsafe { crate::moveit::new::copy(*self).new(this) }
    }

    fn get_ptr(&mut self) -> *mut T {
        unreachable!()
    }
}

impl<T> ValueParam<T> for UniquePtr<T>
where
    T: UniquePtrTarget,
{
    type StackStorage = ();

    fn needs_stack_space(&self) -> bool {
        false
    }

    fn populate_stack_space(&self, _: Pin<&mut MaybeUninit<Self::StackStorage>>) {}

    fn get_ptr(&mut self) -> *mut T {
        (unsafe {
            Pin::into_inner_unchecked(
                self.as_mut()
                    .expect("Passed a NULL UniquePtr as a C++ value parameter"),
            )
        }) as *mut T
    }
}

impl<T> ValueParam<T> for &UniquePtr<T>
where
    T: UniquePtrTarget + CopyNew,
{
    type StackStorage = T;

    fn needs_stack_space(&self) -> bool {
        true
    }

    fn populate_stack_space(&self, this: Pin<&mut MaybeUninit<Self::StackStorage>>) {
        // Invoke a copy constructor on the Rust side, because we'll use std::move on the C++
        // side.
        unsafe {
            crate::moveit::new::copy(
                self.as_ref()
                    .expect("Passed a NULL &UniquePtr as a C++ value parameter"),
            )
            .new(this)
        }
    }

    fn get_ptr(&mut self) -> *mut T {
        unreachable!()
    }
}

/// Implementation detail for how we pass value parameters into C++.
/// This type is instantiated by auto-generated autocxx code each time we
/// need to pass a value parameter into C++, and will take responsibility
/// for extracting that value parameter from the [`ValueParam`] and doing
/// any later cleanup.
#[doc(hidden)]
pub struct ValueParamHandler<T, VP: ValueParam<T>> {
    param: VP,
    space: Option<MaybeUninit<VP::StackStorage>>,
    _pinned: PhantomPinned,
}

impl<T, VP: ValueParam<T>> ValueParamHandler<T, VP> {
    /// Create a new storage space for something that's about to be passed
    /// by value to C++. Depending on the `ValueParam` type passed in,
    /// this may be largely a no-op or it may involve storing a whole
    /// extra copy of the type.
    ///
    /// # Safety
    ///
    /// Callers must guarantee that this type will not move
    /// in memory.
    pub unsafe fn new(param: VP) -> Self {
        let mut this = Self {
            param,
            space: None,
            _pinned: PhantomPinned,
        };
        if this.param.needs_stack_space() {
            this.space = Some(MaybeUninit::uninit());
            this.param
                .populate_stack_space(Pin::new_unchecked(this.space.as_mut().unwrap()));
        }
        this
    }

    /// Return a pointer to the underlying value which can be passed to C++.
    /// Per the unsafety contract of `new`, the object must not have moved
    /// since it was created.
    pub fn get_ptr(&mut self) -> *mut T {
        if let Some(ref mut space) = self.space {
            let ptr =
                unsafe { space.assume_init_mut() } as *mut <VP as ValueParam<T>>::StackStorage;
            unsafe { std::mem::transmute(ptr) }
        } else {
            self.param.get_ptr()
        }
    }
}

impl<T, VP: ValueParam<T>> Drop for ValueParamHandler<T, VP> {
    fn drop(&mut self) {
        if let Some(space) = self.space.take() {
            unsafe { space.assume_init() };
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 DDN. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

use env::{MANAGER_URL, PEM};
use futures::{FutureExt, TryFutureExt};
use iml_agent::{
    agent_error::Result,
    daemon_plugins, env,
    http_comms::{agent_client::AgentClient, crypto_client, session},
    poller, reader,
};

#[tokio::main]
async fn main() -> Result<()> {
    iml_tracing::init();

    tracing::info!("Starting Rust agent_daemon");

    let message_endpoint = MANAGER_URL.join("/agent2/message/")?;

    let start_time = chrono::Utc::now().format("%Y-%m-%dT%T%.6f%:zZ").to_string();

    let identity = crypto_client::get_id(&PEM)?;
    let client = crypto_client::create_client(identity)?;

    let agent_client =
        AgentClient::new(start_time.clone(), message_endpoint.clone(), client.clone());

    let registry = daemon_plugins::plugin_registry();
    let registry_keys: Vec<iml_wire_types::PluginName> = registry.keys().cloned().collect();
    let sessions = session::Sessions::new(&registry_keys);

    tokio::spawn(
        reader::create_reader(sessions.clone(), agent_client.clone(), registry)
            .map_err(|e| {
                tracing::error!("{}", e);
            })
            .map(drop),
    );

    poller::create_poller(agent_client, sessions).await;

    Ok(())
}
<commit_msg>Agent to teardown sessions on signals (#2077)<commit_after>// Copyright (c) 2020 DDN. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

use env::{MANAGER_URL, PEM};
use futures::{
    future::{select, AbortHandle, Abortable},
    FutureExt, TryFutureExt,
};
use iml_agent::{
    agent_error::Result,
    daemon_plugins, env,
    http_comms::{agent_client::AgentClient, crypto_client, session},
    poller, reader,
};
use tokio::signal::unix::{signal, SignalKind};

#[tokio::main]
async fn main() -> Result<()> {
    iml_tracing::init();

    tracing::info!("Starting Rust agent_daemon");

    let message_endpoint = MANAGER_URL.join("/agent2/message/")?;

    let start_time = chrono::Utc::now().format("%Y-%m-%dT%T%.6f%:zZ").to_string();

    let identity = crypto_client::get_id(&PEM)?;
    let client = crypto_client::create_client(identity)?;

    let agent_client =
        AgentClient::new(start_time.clone(), message_endpoint.clone(), client.clone());

    let registry = daemon_plugins::plugin_registry();
    let registry_keys: Vec<iml_wire_types::PluginName> = registry.keys().cloned().collect();
    let sessions = session::Sessions::new(&registry_keys);

    let (reader, reader_reg) = AbortHandle::new_pair();
    tokio::spawn(Abortable::new(
        reader::create_reader(sessions.clone(), agent_client.clone(), registry)
            .map_err(|e| {
                tracing::error!("{}", e);
            })
            .map(drop),
        reader_reg,
    ));

    let (poller, poller_reg) = AbortHandle::new_pair();
    tokio::spawn(Abortable::new(
        poller::create_poller(agent_client, sessions.clone()),
        poller_reg,
    ));

    let mut sigterm = signal(SignalKind::terminate()).expect("Could not listen to SIGTERM");
    let mut sigint = signal(SignalKind::interrupt()).expect("Could not listen to SIGINT");
    select(sigterm.recv().boxed(), sigint.recv().boxed()).await;

    tracing::info!("Terminating on signal...");
    poller.abort();
    reader.abort();
    sessions.terminate_all_sessions().await?;

    Ok(())
}
<|endoftext|>


<commit_before>use frontend::scanner::{scan_into_iterator, Position, ScannerError, Token, TokenWithContext};
use num_traits::{FromPrimitive, ToPrimitive};
use std::iter::Peekable;
use vm::bytecode::{BinaryOp, Chunk, OpCode};

#[derive(Debug)]
pub enum ParsingError {
    /// The end of file has been reached but the parser expected
    /// to find some other token
    UnexpectedEndOfFile,
    /// The parser was expecting a specific token (TODO: we should
    /// report which) but it instead found the reported lexeme
    Unexpected(String, Position),
}

#[derive(Debug)]
pub enum CompilationError {
    ScannerError(ScannerError),
    ParsingError(ParsingError),
}

#[derive(PartialEq, PartialOrd, FromPrimitive, ToPrimitive)]
enum Precedence {
    None,
    Assignment,
    Or,
    And,
    Equality,
    Comparison,
    Term,
    Factor,
    Unary,
    Call,
    Primary,
}

impl Precedence {
    /// Returns the next (as in the immediately higher) Precedence.
    /// Note that this is not defined for the item with the highest
    /// precedence. In such case you'll get a panic
    fn next(&self) -> Precedence {
        // This reduces some boilerplate.
        Precedence::from_u8(self.to_u8().unwrap() + 1).unwrap()
    }
}

/// A single-pass Pratt Parser that consumes tokens from an iterator,
/// parses them into a Lox programs and emits a chunk of bytecode.
/// The parser also keeps tracks of errors.
struct Parser<'a, I>
where
    I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
{
    chunk: &'a mut Chunk,
    tokens: Peekable<I>,
    errors: Vec<CompilationError>,
}

impl<'a, I> Parser<'a, I>
where
    I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
{
    fn new(chunk: &'a mut Chunk, tokens: I) -> Parser<I> {
        Parser {
            chunk: chunk,
            tokens: tokens.peekable(),
            errors: vec![],
        }
    }

    /// Ignores irrelevant (comments and whitespaces) and invalid
    /// tokens.
    /// When invalid tokens are encountered a corresponding error
    /// is generated so that they can be reported and compilation
    /// fails.
    fn skip_to_valid(&mut self) -> () {
        loop {
            match self.tokens.peek() {
                Some(Ok(TokenWithContext {
                    token: Token::Whitespace,
                    lexeme: _,
                    position: _,
                })) => {
                    // No op, just skip it
                }
                Some(Ok(TokenWithContext {
                    token: Token::Comment,
                    lexeme: _,
                    position: _,
                })) => {
                    // No op, just skip it
                }
                Some(Err(ref e)) => {
                    self.errors.push(CompilationError::ScannerError(e.clone()));
                }
                _ => return (),
            }
            let _ = self.tokens.next();
        }
    }

    /// Advances to the next valid token (skipping and keeping track of errors).
    /// This consumes an item from the token stream and returns it.
    fn advance(&mut self) -> Option<TokenWithContext> {
        // Peek to skip errors and keep track of them.
        // This makes unwrapping safe.
        self.skip_to_valid();
        self.tokens.next().map(|r| r.unwrap())
    }

    /// Adds an opcode to the current chunk
    fn emit(&mut self, opcode: OpCode, line: usize) -> () {
        self.chunk.add_instruction(opcode, line)
    }

    /// Peeks the first *valid* token in the iterator
    fn peek(&mut self) -> Option<&TokenWithContext> {
        self.skip_to_valid();
        self.tokens.peek().map(|result| match result {
            Ok(ref token_with_context) => token_with_context,
            Err(_) => unreachable!("We already skipped errors"),
        })
    }

    /// Ensures that a specific token is the next in the input iterator.
    /// If that's the case, it will just consumes it.
    /// If not, it will return a parsing error.
    fn consume(&mut self, token: Token) -> Result<(), ParsingError> {
        let current = self.advance().ok_or(ParsingError::UnexpectedEndOfFile)?;
        if current.token == token {
            Ok(())
        } else {
            Err(ParsingError::Unexpected(
                current.lexeme.clone(),
                current.position,
            ))
        }
    }

    /// Represents the table for the Pratt Parser.
    /// Given the next token it will return a triple containing
    /// (Precedence, Prefix function, Infix function).
    /// The two functions are optional as they might not be specified.
    ///
    /// Note that this function doesn't care about the values possibly
    /// associated with a token. It operates only according to their
    /// variant.
    ///
    /// This is generally used on the token peeked from the front of
    /// the iterator. This lookahead allows us to decide what to do next
    /// according to the "table" below.
    fn find_rule(
        token: &Token,
    ) -> (
        Precedence,
        Option<fn(&mut Self) -> Result<(), ParsingError>>,
        Option<fn(&mut Self) -> Result<(), ParsingError>>,
    )
    where
        I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
    {
        match token {
            Token::LeftParen => (Precedence::Call, Some(Parser::grouping), None),
            Token::RightParen => (Precedence::None, None, None),
            Token::Comma => (Precedence::None, None, None),
            Token::Dot => (Precedence::Call, None, None),
            Token::Minus => (Precedence::Term, Some(Parser::unary), Some(Parser::binary)),
            Token::Plus => (Precedence::Term, None, Some(Parser::binary)),
            Token::Slash => (Precedence::Factor, None, Some(Parser::binary)),
            Token::Star => (Precedence::Factor, None, Some(Parser::binary)),
            Token::Semicolon => (Precedence::None, None, None),
            Token::NumberLiteral(_) => (Precedence::None, Some(Parser::number), None),
            _ => unimplemented!(),
        }
    }

    /// The core of a Pratt Parser.
    /// It peeks tokens to figure out what rule to apply and dispatches the corresponding
    /// functions.
    fn parse_precedence(&mut self, precedence: Precedence) -> Result<(), ParsingError> {
        let prefix_function = {
            let peeked = self.peek().ok_or(ParsingError::UnexpectedEndOfFile)?;
            let (_, prefix_function, _) = Self::find_rule(&peeked.token);
            prefix_function
                .ok_or_else(|| ParsingError::Unexpected(peeked.lexeme.clone(), peeked.position))?
        };
        prefix_function(self)?;
        loop {
            let infix_function = {
                match self.peek() {
                    Some(peeked) => {
                        let (infix_rule_precedence, _, infix_function) =
                            Self::find_rule(&peeked.token);
                        if precedence <= infix_rule_precedence {
                            infix_function.ok_or_else(|| {
                                ParsingError::Unexpected(peeked.lexeme.clone(), peeked.position)
                            })?
                        } else {
                            return Ok(());
                        }
                    }
                    None => return Ok(()),
                }
            };
            infix_function(self)?;
        }
    }

    /// Top level function of the parser.
    /// Parses all the statements in the input and, when necessary,
    /// applies the recovery logic for cleaner error messages.
    fn parse(&mut self) -> Result<(), ()> {
        loop {
            match self.expression() {
                Ok(()) => return Ok(()),
                Err(error) => {
                    self.errors.push(CompilationError::ParsingError(error))
                    //TODO: add recovery mode
                }
            }
        }
    }

    fn expression(&mut self) -> Result<(), ParsingError> {
        self.parse_precedence(Precedence::Assignment)
    }

    fn number(&mut self) -> Result<(), ParsingError> {
        let current = self.advance();
        let (value, line) = if let Some(ref t) = current {
            if let Token::NumberLiteral(ref n) = t.token {
                (*n, t.position.line)
            } else {
                unreachable!()
            }
        } else {
            panic!()
        };
        let constant = self.chunk.add_constant(value);
        self.chunk.add_instruction(OpCode::Constant(constant), line);
        Ok(())
    }

    fn grouping(&mut self) -> Result<(), ParsingError> {
        let _ = self.consume(Token::LeftParen)?;
        self.expression()?;
        self.consume(Token::RightParen)
    }

    fn unary(&mut self) -> Result<(), ParsingError> {
        let (opcode, line) = match self.advance() {
            Some(TokenWithContext {
                token: Token::Minus,
                position,
                lexeme: _,
            }) => (OpCode::Negate, position.line),
            _ => unreachable!("This code is executed only when we know we have a unary expression"),
        };
        let _ = self.parse_precedence(Precedence::Unary)?;
        self.emit(opcode, line);
        Ok(())
    }

    fn binary(&mut self) -> Result<(), ParsingError> {
        let (opcode, line) = match self.advance() {
            Some(TokenWithContext {
                token: Token::Plus,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Add), position.line),
            Some(TokenWithContext {
                token: Token::Minus,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Subtract), position.line),
            Some(TokenWithContext {
                token: Token::Star,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Multiply), position.line),
            Some(TokenWithContext {
                token: Token::Slash,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Divide), position.line),
            _ => {
                unreachable!("This code is executed only when we know we have a binary expression")
            }
        };
        let precedence = {
            let peeked = self
                .peek()
                .ok_or_else(|| ParsingError::UnexpectedEndOfFile)?;
            let (precedence, _, _) = Self::find_rule(&peeked.token);
            precedence.next()
        };
        self.parse_precedence(precedence)?;
        self.emit(opcode, line);
        Ok(())
    }
}

/// Compiles a text producing either the corresponding chunk of bytecode
/// or an error.
/// Error reporting tries to be smart and to minimize reports adopting a
/// "recovery logic".
pub fn compile(text: &str) -> Result<Chunk, Vec<CompilationError>> {
    let mut chunk = Chunk::new();
    let tokens = scan_into_iterator(text);
    {
        let mut parser = Parser::new(&mut chunk, tokens);
        // Chunk is known, errors will be captured later
        // TODO: check if this is the best design
        let _ = parser.parse();
        // TODO: assert that we consumed everything
        // Line is meaningless, but this is temporary to see some results
        // while the implementation is in progress.
        parser.emit(OpCode::Return, 0);
        if parser.errors.len() > 0 {
            return Err(parser.errors);
        }
    }
    Ok(chunk)
}
<commit_msg>Forced consumption of token causing errors<commit_after>use frontend::scanner::{scan_into_iterator, Position, ScannerError, Token, TokenWithContext};
use num_traits::{FromPrimitive, ToPrimitive};
use std::iter::Peekable;
use vm::bytecode::{BinaryOp, Chunk, OpCode};

#[derive(Debug)]
pub enum ParsingError {
    /// The end of file has been reached but the parser expected
    /// to find some other token
    UnexpectedEndOfFile,
    /// The parser was expecting a specific token (TODO: we should
    /// report which) but it instead found the reported lexeme
    Unexpected(String, Position),
}

#[derive(Debug)]
pub enum CompilationError {
    ScannerError(ScannerError),
    ParsingError(ParsingError),
}

#[derive(PartialEq, PartialOrd, FromPrimitive, ToPrimitive)]
enum Precedence {
    None,
    Assignment,
    Or,
    And,
    Equality,
    Comparison,
    Term,
    Factor,
    Unary,
    Call,
    Primary,
}

impl Precedence {
    /// Returns the next (as in the immediately higher) Precedence.
    /// Note that this is not defined for the item with the highest
    /// precedence. In such case you'll get a panic
    fn next(&self) -> Precedence {
        // This reduces some boilerplate.
        Precedence::from_u8(self.to_u8().unwrap() + 1).unwrap()
    }
}

/// A single-pass Pratt Parser that consumes tokens from an iterator,
/// parses them into a Lox programs and emits a chunk of bytecode.
/// The parser also keeps tracks of errors.
struct Parser<'a, I>
where
    I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
{
    chunk: &'a mut Chunk,
    tokens: Peekable<I>,
    errors: Vec<CompilationError>,
}

impl<'a, I> Parser<'a, I>
where
    I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
{
    fn new(chunk: &'a mut Chunk, tokens: I) -> Parser<I> {
        Parser {
            chunk: chunk,
            tokens: tokens.peekable(),
            errors: vec![],
        }
    }

    /// Ignores irrelevant (comments and whitespaces) and invalid
    /// tokens.
    /// When invalid tokens are encountered a corresponding error
    /// is generated so that they can be reported and compilation
    /// fails.
    fn skip_to_valid(&mut self) -> () {
        loop {
            match self.tokens.peek() {
                Some(Ok(TokenWithContext {
                    token: Token::Whitespace,
                    lexeme: _,
                    position: _,
                })) => {
                    // No op, just skip it
                }
                Some(Ok(TokenWithContext {
                    token: Token::Comment,
                    lexeme: _,
                    position: _,
                })) => {
                    // No op, just skip it
                }
                Some(Err(ref e)) => {
                    self.errors.push(CompilationError::ScannerError(e.clone()));
                }
                _ => return (),
            }
            let _ = self.tokens.next();
        }
    }

    /// Advances to the next valid token (skipping and keeping track of errors).
    /// This consumes an item from the token stream and returns it.
    fn advance(&mut self) -> Option<TokenWithContext> {
        // Peek to skip errors and keep track of them.
        // This makes unwrapping safe.
        self.skip_to_valid();
        self.tokens.next().map(|r| r.unwrap())
    }

    /// Adds an opcode to the current chunk
    fn emit(&mut self, opcode: OpCode, line: usize) -> () {
        self.chunk.add_instruction(opcode, line)
    }

    /// Peeks the first *valid* token in the iterator
    fn peek(&mut self) -> Option<&TokenWithContext> {
        self.skip_to_valid();
        self.tokens.peek().map(|result| match result {
            Ok(ref token_with_context) => token_with_context,
            Err(_) => unreachable!("We already skipped errors"),
        })
    }

    /// Ensures that a specific token is the next in the input iterator.
    /// If that's the case, it will just consumes it.
    /// If not, it will return a parsing error.
    fn consume(&mut self, token: Token) -> Result<(), ParsingError> {
        let current = self.advance().ok_or(ParsingError::UnexpectedEndOfFile)?;
        if current.token == token {
            Ok(())
        } else {
            Err(ParsingError::Unexpected(
                current.lexeme.clone(),
                current.position,
            ))
        }
    }

    /// Represents the table for the Pratt Parser.
    /// Given the next token it will return a triple containing
    /// (Precedence, Prefix function, Infix function).
    /// The two functions are optional as they might not be specified.
    ///
    /// Note that this function doesn't care about the values possibly
    /// associated with a token. It operates only according to their
    /// variant.
    ///
    /// This is generally used on the token peeked from the front of
    /// the iterator. This lookahead allows us to decide what to do next
    /// according to the "table" below.
    fn find_rule(
        token: &Token,
    ) -> (
        Precedence,
        Option<fn(&mut Self) -> Result<(), ParsingError>>,
        Option<fn(&mut Self) -> Result<(), ParsingError>>,
    )
    where
        I: Iterator<Item = Result<TokenWithContext, ScannerError>>,
    {
        match token {
            Token::LeftParen => (Precedence::Call, Some(Parser::grouping), None),
            Token::RightParen => (Precedence::None, None, None),
            Token::Comma => (Precedence::None, None, None),
            Token::Dot => (Precedence::Call, None, None),
            Token::Minus => (Precedence::Term, Some(Parser::unary), Some(Parser::binary)),
            Token::Plus => (Precedence::Term, None, Some(Parser::binary)),
            Token::Slash => (Precedence::Factor, None, Some(Parser::binary)),
            Token::Star => (Precedence::Factor, None, Some(Parser::binary)),
            Token::Semicolon => (Precedence::None, None, None),
            Token::NumberLiteral(_) => (Precedence::None, Some(Parser::number), None),
            _ => unimplemented!(),
        }
    }

    /// The core of a Pratt Parser.
    /// It peeks tokens to figure out what rule to apply and dispatches the corresponding
    /// functions.
    fn parse_precedence(&mut self, precedence: Precedence) -> Result<(), ParsingError> {
        let prefix_function = {
            let (_, prefix_function, _) = {
                let peeked = self.peek().ok_or(ParsingError::UnexpectedEndOfFile)?;
                Self::find_rule(&peeked.token)
            };
            prefix_function.ok_or_else(|| {
                // This is a bad token. We need to consume it so we can carry on.
                let token = self.advance().unwrap();
                ParsingError::Unexpected(token.lexeme.clone(), token.position)
            })?
        };
        prefix_function(self)?;
        loop {
            let infix_function = {
                let (infix_rule_precedence, _, infix_function) = {
                    match self.peek() {
                        Some(peeked) => Self::find_rule(&peeked.token),
                        None => return Ok(()),
                    }
                };
                if precedence <= infix_rule_precedence {
                    infix_function.ok_or_else(|| {
                        // This is a bad token. We need to consume it so we can carry on.
                        let token = self.advance().unwrap();
                        ParsingError::Unexpected(token.lexeme.clone(), token.position)
                    })?
                } else {
                    return Ok(());
                }
            };
            infix_function(self)?;
        }
    }

    /// Top level function of the parser.
    /// Parses all the statements in the input and, when necessary,
    /// applies the recovery logic for cleaner error messages.
    fn parse(&mut self) -> Result<(), ()> {
        while let Some(_) = self.peek() {
            if let Err(error) = self.expression() {
                self.errors.push(CompilationError::ParsingError(error));
            }
        }
        if self.errors.len() > 0 {
            Err(())
        } else {
            Ok(())
        }
    }

    fn expression(&mut self) -> Result<(), ParsingError> {
        self.parse_precedence(Precedence::Assignment)
    }

    fn number(&mut self) -> Result<(), ParsingError> {
        let current = self.advance();
        let (value, line) = if let Some(ref t) = current {
            if let Token::NumberLiteral(ref n) = t.token {
                (*n, t.position.line)
            } else {
                unreachable!()
            }
        } else {
            panic!()
        };
        let constant = self.chunk.add_constant(value);
        self.chunk.add_instruction(OpCode::Constant(constant), line);
        Ok(())
    }

    fn grouping(&mut self) -> Result<(), ParsingError> {
        let _ = self.consume(Token::LeftParen)?;
        self.expression()?;
        self.consume(Token::RightParen)
    }

    fn unary(&mut self) -> Result<(), ParsingError> {
        let (opcode, line) = match self.advance() {
            Some(TokenWithContext {
                token: Token::Minus,
                position,
                lexeme: _,
            }) => (OpCode::Negate, position.line),
            _ => unreachable!("This code is executed only when we know we have a unary expression"),
        };
        let _ = self.parse_precedence(Precedence::Unary)?;
        self.emit(opcode, line);
        Ok(())
    }

    fn binary(&mut self) -> Result<(), ParsingError> {
        let (opcode, line) = match self.advance() {
            Some(TokenWithContext {
                token: Token::Plus,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Add), position.line),
            Some(TokenWithContext {
                token: Token::Minus,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Subtract), position.line),
            Some(TokenWithContext {
                token: Token::Star,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Multiply), position.line),
            Some(TokenWithContext {
                token: Token::Slash,
                position,
                lexeme: _,
            }) => (OpCode::Binary(BinaryOp::Divide), position.line),
            _ => {
                unreachable!("This code is executed only when we know we have a binary expression")
            }
        };
        let precedence = {
            let peeked = self
                .peek()
                .ok_or_else(|| ParsingError::UnexpectedEndOfFile)?;
            let (precedence, _, _) = Self::find_rule(&peeked.token);
            precedence.next()
        };
        self.parse_precedence(precedence)?;
        self.emit(opcode, line);
        Ok(())
    }
}

/// Compiles a text producing either the corresponding chunk of bytecode
/// or an error.
/// Error reporting tries to be smart and to minimize reports adopting a
/// "recovery logic".
pub fn compile(text: &str) -> Result<Chunk, Vec<CompilationError>> {
    let mut chunk = Chunk::new();
    let tokens = scan_into_iterator(text);
    {
        let mut parser = Parser::new(&mut chunk, tokens);
        // Chunk is known, errors will be captured later
        // TODO: check if this is the best design
        let _ = parser.parse();
        // TODO: assert that we consumed everything
        // Line is meaningless, but this is temporary to see some results
        // while the implementation is in progress.
        parser.emit(OpCode::Return, 0);
        if parser.errors.len() > 0 {
            return Err(parser.errors);
        }
    }
    Ok(chunk)
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at https://mozilla.org/MPL/2.0/.

use zip::Zip;
use errors::Result;
use errors::ResultExt;

use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::io::Read;
use std::io::Write;
use std::fs;
use std::fs::DirBuilder;
use std::fs::File;
use std::process::Command;

use tempdir::TempDir;

/// Zip files using the system `zip` command.
///
/// Create a temporary directory, write temp files in that directory, and then
/// calls the zip command to generate an epub file.
///
/// This method will fail if `zip` (or the alternate specified command) is not installed
/// on the user system.
///
/// Note that these takes care of adding the mimetype (since it must not be deflated), it
/// should not be added manually.
pub struct ZipCommand {
    command: String,
    temp_dir: TempDir,
    files: Vec<PathBuf>,
}

impl ZipCommand {
    /// Creates a new ZipCommand, using default setting to create a temporary directory.
    pub fn new() -> Result<ZipCommand> {
        let temp_dir = TempDir::new("epub").chain_err(|| "could not create temporary directory")?;
        let zip = ZipCommand {
            command: String::from("zip"),
            temp_dir: temp_dir,
            files: vec![],
        };
        Ok(zip)
    }

    /// Creates a new ZipCommand, specifying where to create a temporary directory.
    ///
    /// # Arguments
    /// * `temp_path`: the path where a temporary directory should be created.
    pub fn new_in<P: AsRef<Path>>(temp_path: P) -> Result<ZipCommand> {
        let temp_dir = TempDir::new_in(temp_path, "epub")
            .chain_err(|| "could not create temporary directory")?;
        let zip = ZipCommand {
            command: String::from("zip"),
            temp_dir: temp_dir,
            files: vec![],
        };
        Ok(zip)
    }

    /// Set zip command to use (default: "zip")
    pub fn command<S: Into<String>>(&mut self, command: S) -> &mut Self {
        self.command = command.into();
        self
    }

    /// Adds a file to the temporary directory
    fn add_to_tmp_dir<P: AsRef<Path>, R: Read>(&mut self, path: P, mut content: R) -> Result<()> {
        let dest_file = self.temp_dir
            .path()
            .join(path.as_ref());
        let dest_dir = dest_file.parent().unwrap();
        if !fs::metadata(dest_dir).is_ok() {
            // dir does not exist, create it
            DirBuilder::new().recursive(true)
                .create(&dest_dir)
                .chain_err(|| {
                    format!("could not create temporary directory in {path}",
                            path = dest_dir.display())
                })?;
        }

        let mut f = File::create(&dest_file).chain_err(|| {
                format!("could not write to temporary file {file}",
                        file = path.as_ref().display())
            })?;
        io::copy(&mut content, &mut f).chain_err(|| {
                format!("could not write to temporary file {file}",
                        file = path.as_ref().display())
            })?;
        Ok(())
    }
}

impl Zip for ZipCommand {
    fn write_file<P: AsRef<Path>, R: Read>(&mut self, path: P, content: R) -> Result<()> {
        let path = path.as_ref();
        if path.starts_with("..") || path.is_absolute() {
            bail!("file {file} refers to a path outside the temporary directory. This is \
                   verbotten!");
        }

        self.add_to_tmp_dir(path, content)?;
        self.files.push(path.to_path_buf());
        Ok(())
    }

    fn generate<W: Write>(&mut self, mut to: W) -> Result<()> {
        let mut command = Command::new(&self.command);

        // First, add mimetype and don't compress it
        self.add_to_tmp_dir("mimetype", b"application/epub+zip".as_ref())?;
        let output = command.current_dir(self.temp_dir.path())
            .arg("-X0")
            .arg("output.epub")
            .arg("mimetype")
            .output()
            .chain_err(|| format!("failed to run command {name}", name = self.command))?;
        if !output.status.success() {
            bail!("command {name} didn't return succesfully: {output}",
                  name = self.command,
                  output = String::from_utf8_lossy(&output.stderr));
        }

        command.current_dir(self.temp_dir.path())
            .arg("-X")
            .arg("output.epub");
        for file in &self.files {
            command.arg(format!("{}", file.display()));
        }

        let output = command.output()
            .chain_err(|| format!("failed to run command {name}", name = self.command))?;
        if output.status.success() {
            let mut f = File::open(self.temp_dir.path().join("output.epub"))
                .chain_err(|| "error reading temporary epub file")?;
            io::copy(&mut f, &mut to)
                .chain_err(|| "error writing result of the zip command")?;
            println!("{}", String::from_utf8_lossy(&output.stderr));
            Ok(())
        } else {
            bail!("command {name} didn't return succesfully: {output}",
                  name = self.command,
                  output = String::from_utf8_lossy(&output.stderr));
        }
    }
}
<commit_msg>Deflate all files but mimetype when using zip command<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at https://mozilla.org/MPL/2.0/.

use zip::Zip;
use errors::Result;
use errors::ResultExt;

use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::io::Read;
use std::io::Write;
use std::fs;
use std::fs::DirBuilder;
use std::fs::File;
use std::process::Command;

use tempdir::TempDir;

/// Zip files using the system `zip` command.
///
/// Create a temporary directory, write temp files in that directory, and then
/// calls the zip command to generate an epub file.
///
/// This method will fail if `zip` (or the alternate specified command) is not installed
/// on the user system.
///
/// Note that these takes care of adding the mimetype (since it must not be deflated), it
/// should not be added manually.
pub struct ZipCommand {
    command: String,
    temp_dir: TempDir,
    files: Vec<PathBuf>,
}

impl ZipCommand {
    /// Creates a new ZipCommand, using default setting to create a temporary directory.
    pub fn new() -> Result<ZipCommand> {
        let temp_dir = TempDir::new("epub").chain_err(|| "could not create temporary directory")?;
        let zip = ZipCommand {
            command: String::from("zip"),
            temp_dir: temp_dir,
            files: vec![],
        };
        Ok(zip)
    }

    /// Creates a new ZipCommand, specifying where to create a temporary directory.
    ///
    /// # Arguments
    /// * `temp_path`: the path where a temporary directory should be created.
    pub fn new_in<P: AsRef<Path>>(temp_path: P) -> Result<ZipCommand> {
        let temp_dir = TempDir::new_in(temp_path, "epub")
            .chain_err(|| "could not create temporary directory")?;
        let zip = ZipCommand {
            command: String::from("zip"),
            temp_dir: temp_dir,
            files: vec![],
        };
        Ok(zip)
    }

    /// Set zip command to use (default: "zip")
    pub fn command<S: Into<String>>(&mut self, command: S) -> &mut Self {
        self.command = command.into();
        self
    }

    /// Adds a file to the temporary directory
    fn add_to_tmp_dir<P: AsRef<Path>, R: Read>(&mut self, path: P, mut content: R) -> Result<()> {
        let dest_file = self.temp_dir
            .path()
            .join(path.as_ref());
        let dest_dir = dest_file.parent().unwrap();
        if !fs::metadata(dest_dir).is_ok() {
            // dir does not exist, create it
            DirBuilder::new().recursive(true)
                .create(&dest_dir)
                .chain_err(|| {
                    format!("could not create temporary directory in {path}",
                            path = dest_dir.display())
                })?;
        }

        let mut f = File::create(&dest_file).chain_err(|| {
                format!("could not write to temporary file {file}",
                        file = path.as_ref().display())
            })?;
        io::copy(&mut content, &mut f).chain_err(|| {
                format!("could not write to temporary file {file}",
                        file = path.as_ref().display())
            })?;
        Ok(())
    }
}

impl Zip for ZipCommand {
    fn write_file<P: AsRef<Path>, R: Read>(&mut self, path: P, content: R) -> Result<()> {
        let path = path.as_ref();
        if path.starts_with("..") || path.is_absolute() {
            bail!("file {file} refers to a path outside the temporary directory. This is \
                   verbotten!");
        }

        self.add_to_tmp_dir(path, content)?;
        self.files.push(path.to_path_buf());
        Ok(())
    }

    fn generate<W: Write>(&mut self, mut to: W) -> Result<()> {
        let mut command = Command::new(&self.command);

        // First, add mimetype and don't compress it
        self.add_to_tmp_dir("mimetype", b"application/epub+zip".as_ref())?;
        let output = Command::new(&self.command)
            .current_dir(self.temp_dir.path())
            .arg("-X0")
            .arg("output.epub")
            .arg("mimetype")
            .output()
            .chain_err(|| format!("failed to run command {name}", name = self.command))?;
        if !output.status.success() {
            bail!("command {name} didn't return succesfully: {output}",
                  name = self.command,
                  output = String::from_utf8_lossy(&output.stderr));
        }

        command.current_dir(self.temp_dir.path())
            .arg("-9")
            .arg("output.epub");
        for file in &self.files {
            command.arg(format!("{}", file.display()));
        }

        let output = command.output()
            .chain_err(|| format!("failed to run command {name}", name = self.command))?;
        if output.status.success() {
            let mut f = File::open(self.temp_dir.path().join("output.epub"))
                .chain_err(|| "error reading temporary epub file")?;
            io::copy(&mut f, &mut to)
                .chain_err(|| "error writing result of the zip command")?;
            Ok(())
        } else {
            bail!("command {name} didn't return succesfully: {output}",
                  name = self.command,
                  output = String::from_utf8_lossy(&output.stderr));
        }
    }
}
<|endoftext|>


<commit_before>use libc::{c_int, c_ushort, c_uchar, size_t};

use symbols::{ZOPFLI_WINDOW_MASK, ZOPFLI_MIN_MATCH};

const HASH_SHIFT: c_int = 5;
const HASH_MASK: c_int = 32767;

#[repr(C)]
pub struct ZopfliHash {
    head: Vec<c_int>,  /* Hash value to index of its most recent occurrence. */
    prev: Vec<c_ushort>,  /* Index to index of prev. occurrence of same hash. */
    hashval: Vec<c_int>,  /* Index to hash value at this index. */
    val: c_int,  /* Current hash value. */

    /* Fields with similar purpose as the above hash, but for the second hash with
    a value that is calculated differently.  */
    head2: Vec<c_int>,  /* Hash value to index of its most recent occurrence. */
    prev2: Vec<c_ushort>,  /* Index to index of prev. occurrence of same hash. */
    hashval2: Vec<c_int>,  /* Index to hash value at this index. */
    val2: c_int,  /* Current hash value. */

    same: Vec<c_ushort>,  /* Amount of repetitions of same byte after this .*/
}

impl ZopfliHash {
    pub fn new(window_size: size_t) -> ZopfliHash {
        ZopfliHash {
           head: vec![-1; 65536],
           prev: (0..window_size as c_ushort).collect::<Vec<_>>(),
           hashval: vec![-1; window_size],
           val: 0,

           /* Fields with similar purpose as the above hash, but for the second hash with
           a value that is calculated differently.  */
           head2: vec![-1; 65536],
           prev2: (0..window_size as c_ushort).collect::<Vec<_>>(),
           hashval2: vec![-1; window_size],
           val2: 0,

           same: vec![0; window_size],
       }
    }
}

/// Update the sliding hash value with the given byte. All calls to this function
/// must be made on consecutive input characters. Since the hash value exists out
/// of multiple input bytes, a few warmups with this function are needed initially.
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn UpdateHashValue(h_ptr: *mut ZopfliHash, c: c_uchar) {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.val = ((h.val << HASH_SHIFT) ^ c as c_int) & HASH_MASK;
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliWarmupHash(array: *const c_uchar, pos: size_t, end: size_t, h_ptr: *mut ZopfliHash) {
    unsafe {
        UpdateHashValue(h_ptr, *array.offset((pos + 0) as isize));
    }
    if pos + 1 < end {
        unsafe {
            UpdateHashValue(h_ptr, *array.offset((pos + 1) as isize));
        }
    }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliUpdateHash(array: *const c_uchar, pos: size_t, end: size_t, h_ptr: *mut ZopfliHash) {
    let hpos = (pos & ZOPFLI_WINDOW_MASK) as usize;
    let mut amount: c_int = 0;

    let hash_value = if pos + ZOPFLI_MIN_MATCH as size_t <= end {
        unsafe { *array.offset((pos + ZOPFLI_MIN_MATCH as size_t - 1) as isize) }
    } else {
        0
    };
    UpdateHashValue(h_ptr, hash_value);

    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval[hpos] = h.val;

    if h.head[h.val as usize] != -1 && h.hashval[h.head[h.val as usize] as usize] == h.val {
        h.prev[hpos] = h.head[h.val as usize] as c_ushort;
    } else {
        h.prev[hpos] = hpos as c_ushort;
    }

    h.head[h.val as usize] = hpos as c_int;

    /* Update "same". */
    if h.same[((pos - 1) & ZOPFLI_WINDOW_MASK) as usize] > 1 {
        amount = h.same[((pos - 1) & ZOPFLI_WINDOW_MASK) as usize] as c_int - 1;
    }

    unsafe {
        while pos + amount as size_t + 1 < end && *array.offset(pos as isize) == *array.offset((pos + amount as size_t + 1) as isize) && amount < -1 {
            amount += 1;
        }
    }
    h.same[hpos] = amount as c_ushort;

    h.val2 = (((h.same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h.val as c_ushort) as c_int;
    h.hashval2[hpos] = h.val2;
    if h.head2[h.val2 as usize] != -1 as i32 && h.hashval2[h.head2[h.val2 as usize] as usize] == h.val2 {
        h.prev2[hpos] = h.head2[h.val2 as usize] as c_ushort;
    } else {
        h.prev2[hpos] = hpos as c_ushort;
    }
    h.head2[h.val2 as usize] = hpos as c_int;
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHead(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.head.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashPrev(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.prev.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHashval(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashVal(h_ptr: *mut ZopfliHash) -> c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.val
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHead2(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.head2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashPrev2(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.prev2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHashval2(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashVal2(h_ptr: *mut ZopfliHash) -> c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.val2
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashSame(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.same.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliCleanHash(ptr: *mut ZopfliHash) {
    if ptr.is_null() { return }
    unsafe { Box::from_raw(ptr); }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliInitHash(window_size: size_t) -> *mut ZopfliHash {
    Box::into_raw(Box::new(ZopfliHash::new(window_size)))
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliResetHash(window_size: size_t, h_ptr: *mut ZopfliHash) {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };

    h.val = 0;
    h.head = vec![-1; 65536];
    h.prev = (0..window_size as c_ushort).collect::<Vec<_>>();
    h.hashval = vec![-1; window_size];

    h.same = vec![0; window_size];

    h.val2 = 0;
    h.head2 = vec![-1; 65536];
    h.prev2 = (0..window_size as c_ushort).collect::<Vec<_>>();
    h.hashval2 = vec![-1; window_size];
}
<commit_msg>Make UpdateHashValue's work be a method on ZopfliHash<commit_after>use libc::{c_int, c_ushort, c_uchar, size_t};

use symbols::{ZOPFLI_WINDOW_MASK, ZOPFLI_MIN_MATCH};

const HASH_SHIFT: c_int = 5;
const HASH_MASK: c_int = 32767;

#[repr(C)]
pub struct ZopfliHash {
    head: Vec<c_int>,  /* Hash value to index of its most recent occurrence. */
    prev: Vec<c_ushort>,  /* Index to index of prev. occurrence of same hash. */
    hashval: Vec<c_int>,  /* Index to hash value at this index. */
    val: c_int,  /* Current hash value. */

    /* Fields with similar purpose as the above hash, but for the second hash with
    a value that is calculated differently.  */
    head2: Vec<c_int>,  /* Hash value to index of its most recent occurrence. */
    prev2: Vec<c_ushort>,  /* Index to index of prev. occurrence of same hash. */
    hashval2: Vec<c_int>,  /* Index to hash value at this index. */
    val2: c_int,  /* Current hash value. */

    same: Vec<c_ushort>,  /* Amount of repetitions of same byte after this .*/
}

impl ZopfliHash {
    pub fn new(window_size: size_t) -> ZopfliHash {
        ZopfliHash {
           head: vec![-1; 65536],
           prev: (0..window_size as c_ushort).collect::<Vec<_>>(),
           hashval: vec![-1; window_size],
           val: 0,

           /* Fields with similar purpose as the above hash, but for the second hash with
           a value that is calculated differently.  */
           head2: vec![-1; 65536],
           prev2: (0..window_size as c_ushort).collect::<Vec<_>>(),
           hashval2: vec![-1; window_size],
           val2: 0,

           same: vec![0; window_size],
       }
    }

    /// Update the sliding hash value with the given byte. All calls to this function
    /// must be made on consecutive input characters. Since the hash value exists out
    /// of multiple input bytes, a few warmups with this function are needed initially.
    pub fn update(&mut self, c: c_uchar) {
        self.val = ((self.val << HASH_SHIFT) ^ c as c_int) & HASH_MASK;
    }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn UpdateHashValue(h_ptr: *mut ZopfliHash, c: c_uchar) {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.update(c);
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliWarmupHash(array: *const c_uchar, pos: size_t, end: size_t, h_ptr: *mut ZopfliHash) {
    unsafe {
        UpdateHashValue(h_ptr, *array.offset((pos + 0) as isize));
    }
    if pos + 1 < end {
        unsafe {
            UpdateHashValue(h_ptr, *array.offset((pos + 1) as isize));
        }
    }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliUpdateHash(array: *const c_uchar, pos: size_t, end: size_t, h_ptr: *mut ZopfliHash) {
    let hpos = (pos & ZOPFLI_WINDOW_MASK) as usize;
    let mut amount: c_int = 0;

    let hash_value = if pos + ZOPFLI_MIN_MATCH as size_t <= end {
        unsafe { *array.offset((pos + ZOPFLI_MIN_MATCH as size_t - 1) as isize) }
    } else {
        0
    };
    UpdateHashValue(h_ptr, hash_value);

    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval[hpos] = h.val;

    if h.head[h.val as usize] != -1 && h.hashval[h.head[h.val as usize] as usize] == h.val {
        h.prev[hpos] = h.head[h.val as usize] as c_ushort;
    } else {
        h.prev[hpos] = hpos as c_ushort;
    }

    h.head[h.val as usize] = hpos as c_int;

    /* Update "same". */
    if h.same[((pos - 1) & ZOPFLI_WINDOW_MASK) as usize] > 1 {
        amount = h.same[((pos - 1) & ZOPFLI_WINDOW_MASK) as usize] as c_int - 1;
    }

    unsafe {
        while pos + amount as size_t + 1 < end && *array.offset(pos as isize) == *array.offset((pos + amount as size_t + 1) as isize) && amount < -1 {
            amount += 1;
        }
    }
    h.same[hpos] = amount as c_ushort;

    h.val2 = (((h.same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h.val as c_ushort) as c_int;
    h.hashval2[hpos] = h.val2;
    if h.head2[h.val2 as usize] != -1 as i32 && h.hashval2[h.head2[h.val2 as usize] as usize] == h.val2 {
        h.prev2[hpos] = h.head2[h.val2 as usize] as c_ushort;
    } else {
        h.prev2[hpos] = hpos as c_ushort;
    }
    h.head2[h.val2 as usize] = hpos as c_int;
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHead(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.head.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashPrev(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.prev.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHashval(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashVal(h_ptr: *mut ZopfliHash) -> c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.val
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHead2(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.head2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashPrev2(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.prev2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashHashval2(h_ptr: *mut ZopfliHash) -> *mut c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.hashval2.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashVal2(h_ptr: *mut ZopfliHash) -> c_int {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.val2
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliHashSame(h_ptr: *mut ZopfliHash) -> *mut c_ushort {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };
    h.same.as_mut_ptr()
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliCleanHash(ptr: *mut ZopfliHash) {
    if ptr.is_null() { return }
    unsafe { Box::from_raw(ptr); }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliInitHash(window_size: size_t) -> *mut ZopfliHash {
    Box::into_raw(Box::new(ZopfliHash::new(window_size)))
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ZopfliResetHash(window_size: size_t, h_ptr: *mut ZopfliHash) {
    let h = unsafe {
        assert!(!h_ptr.is_null());
        &mut *h_ptr
    };

    h.val = 0;
    h.head = vec![-1; 65536];
    h.prev = (0..window_size as c_ushort).collect::<Vec<_>>();
    h.hashval = vec![-1; window_size];

    h.same = vec![0; window_size];

    h.val2 = 0;
    h.head2 = vec![-1; 65536];
    h.prev2 = (0..window_size as c_ushort).collect::<Vec<_>>();
    h.hashval2 = vec![-1; window_size];
}
<|endoftext|>