<commit_before>// Copyright 2018 Google LLC
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT.

/// - The PubSub server sets up TCP listeners on 2 ports, the "subscriber" port and the "publisher"
///   port. Because both publishers and subscribers initiate their connections to the PubSub
///   server, the server requires no prior knowledge of either publishers or subscribers.
///
/// - Subscribers connect to the server on the server's "subscriber" port. Once the connection is
///   established, the server acts as the client, sending messages to the subscribers via a
///   Subscriber service.
///
/// - Publishers connect to the server on the "publisher"port, and once connected, they send
///   messages to the server to be publish via a Publisher service.
///
///       Subscriber                        Publisher                       PubSub Server
/// T1        |                                 |                                 |
/// T2        |-----Connect------------------------------------------------------>|
/// T3        |                                 |                                 |
/// T4        |                                 |-----Connect-------------------->|
/// T5        |                                 |                                 |
/// T6        |                                 |-----Publish-------------------->|
/// T7        |                                 |                                 |
/// T8        |<------------------------------------------------------Receive-----|
/// T9        |-----(Receive OK)------------------------------------------------->|
/// T10       |                                 |                                 |
/// T11       |                                 |<--------------(Publish OK)------|
use anyhow::anyhow;
use futures::{
    channel::oneshot,
    future::{self, AbortHandle},
    prelude::*,
};
use log::info;
use publisher::Publisher as _;
use std::{
    collections::HashMap,
    io,
    net::SocketAddr,
    sync::{Arc, Mutex, RwLock},
};
use subscriber::Subscriber as _;
use tarpc::{
    client, context,
    serde_transport::tcp,
    server::{self, Channel},
};
use tokio::net::ToSocketAddrs;
use tokio_serde::formats::Json;

pub mod subscriber {
    #[tarpc::service]
    pub trait Subscriber {
        async fn topics() -> Vec<String>;
        async fn receive(topic: String, message: String);
    }
}

pub mod publisher {
    #[tarpc::service]
    pub trait Publisher {
        async fn publish(topic: String, message: String);
    }
}

#[derive(Clone, Debug)]
struct Subscriber {
    local_addr: SocketAddr,
    topics: Vec<String>,
}

#[tarpc::server]
impl subscriber::Subscriber for Subscriber {
    async fn topics(self, _: context::Context) -> Vec<String> {
        self.topics.clone()
    }

    async fn receive(self, _: context::Context, topic: String, message: String) {
        info!(
            "[{}] received message on topic '{}': {}",
            self.local_addr, topic, message
        );
    }
}

struct SubscriberHandle(AbortHandle);

impl Drop for SubscriberHandle {
    fn drop(&mut self) {
        self.0.abort();
    }
}

impl Subscriber {
    async fn connect(
        publisher_addr: impl ToSocketAddrs,
        topics: Vec<String>,
    ) -> anyhow::Result<SubscriberHandle> {
        let publisher = tcp::connect(publisher_addr, Json::default()).await?;
        let local_addr = publisher.local_addr()?;
        let mut handler = server::BaseChannel::with_defaults(publisher)
            .respond_with(Subscriber { local_addr, topics }.serve());
        // The first request is for the topics being subscriibed to.
        match handler.next().await {
            Some(init_topics) => init_topics?.await,
            None => {
                return Err(anyhow!(
                    "[{}] Server never initialized the subscriber.",
                    local_addr
                ))
            }
        };
        let (handler, abort_handle) = future::abortable(handler.execute());
        tokio::spawn(async move {
            match handler.await {
                Ok(()) | Err(future::Aborted) => info!("[{}] subscriber shutdown.", local_addr),
            }
        });
        Ok(SubscriberHandle(abort_handle))
    }
}

#[derive(Debug)]
struct Subscription {
    subscriber: subscriber::SubscriberClient,
    topics: Vec<String>,
}

#[derive(Clone, Debug)]
struct Publisher {
    clients: Arc<Mutex<HashMap<SocketAddr, Subscription>>>,
    subscriptions: Arc<RwLock<HashMap<String, HashMap<SocketAddr, subscriber::SubscriberClient>>>>,
}

struct PublisherAddrs {
    publisher: SocketAddr,
    subscriptions: SocketAddr,
}

impl Publisher {
    async fn start(self) -> io::Result<PublisherAddrs> {
        let mut connecting_publishers = tcp::listen("localhost:0", Json::default).await?;

        let publisher_addrs = PublisherAddrs {
            publisher: connecting_publishers.local_addr(),
            subscriptions: self.clone().start_subscription_manager().await?,
        };

        info!("[{}] listening for publishers.", publisher_addrs.publisher);
        tokio::spawn(async move {
            // Because this is just an example, we know there will only be one publisher. In more
            // realistic code, this would be a loop to continually accept new publisher
            // connections.
            let publisher = connecting_publishers.next().await.unwrap().unwrap();
            info!("[{}] publisher connected.", publisher.peer_addr().unwrap());

            server::BaseChannel::with_defaults(publisher)
                .respond_with(self.serve())
                .execute()
                .await
        });

        Ok(publisher_addrs)
    }

    async fn start_subscription_manager(self) -> io::Result<SocketAddr> {
        let mut connecting_subscribers = tcp::listen("localhost:0", Json::default)
            .await?
            .filter_map(|r| future::ready(r.ok()));
        let new_subscriber_addr = connecting_subscribers.get_ref().local_addr();
        info!("[{}] listening for subscribers.", new_subscriber_addr);

        tokio::spawn(async move {
            while let Some(conn) = connecting_subscribers.next().await {
                let subscriber_addr = conn.peer_addr().unwrap();

                let tarpc::client::NewClient {
                    client: mut subscriber,
                    dispatch,
                } = subscriber::SubscriberClient::new(client::Config::default(), conn);
                self.clients.lock().unwrap().insert(
                    subscriber_addr,
                    Subscription {
                        subscriber: subscriber.clone(),
                        topics: Vec::new(),
                    },
                );

                let (ready_tx, ready) = oneshot::channel();
                self.clone()
                    .start_subscriber_gc(subscriber_addr, dispatch, ready);

                // Populate the topics
                if let Ok(topics) = subscriber.topics(context::current()).await {
                    info!("[{}] subscribed to topics: {:?}", subscriber_addr, topics);
                    let mut subscriptions = self.subscriptions.write().unwrap();
                    for topic in topics {
                        subscriptions
                            .entry(topic)
                            .or_insert_with(HashMap::new)
                            .insert(subscriber_addr, subscriber.clone());
                    }
                }
                // Signal that initialization is done.
                ready_tx.send(()).unwrap();
            }
        });

        Ok(new_subscriber_addr)
    }

    fn start_subscriber_gc(
        self,
        subscriber_addr: SocketAddr,
        client_dispatch: impl Future<Output = anyhow::Result<()>> + Send + 'static,
        subscriber_ready: oneshot::Receiver<()>,
    ) {
        tokio::spawn(async move {
            match client_dispatch.await {
                Ok(()) => info!("[{:?}] subscriber connection closed", subscriber_addr),
                Err(e) => info!(
                    "[{:?}] subscriber connection broken: {:?}",
                    subscriber_addr, e
                ),
            }
            // Don't clean up the subscriber until initialization is done.
            let _ = subscriber_ready.await;
            if let Some(subscription) = self.clients.lock().unwrap().remove(&subscriber_addr) {
                let mut subscriptions = self.subscriptions.write().unwrap();
                for topic in subscription.topics {
                    let subscribers = subscriptions.get_mut(&topic).unwrap();
                    subscribers.remove(&subscriber_addr);
                    if subscribers.is_empty() {
                        subscriptions.remove(&topic);
                    }
                }
            }
        });
    }
}

#[tarpc::server]
impl publisher::Publisher for Publisher {
    async fn publish(self, _: context::Context, topic: String, message: String) {
        info!("received message to publish.");
        let mut subscribers = match self.subscriptions.read().unwrap().get(&topic) {
            None => return,
            Some(subscriptions) => subscriptions.clone(),
        };
        let mut publications = Vec::new();
        for client in subscribers.values_mut() {
            publications.push(client.receive(context::current(), topic.clone(), message.clone()));
        }
        // Ignore failing subscribers. In a real pubsub, you'd want to continually retry until
        // subscribers ack. Of course, a lot would be different in a real pubsub :)
        future::join_all(publications).await;
    }
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    env_logger::init();

    let clients = Arc::new(Mutex::new(HashMap::new()));
    let addrs = Publisher {
        clients,
        subscriptions: Arc::new(RwLock::new(HashMap::new())),
    }
    .start()
    .await?;

    let _subscriber0 = Subscriber::connect(
        addrs.subscriptions,
        vec!["calculus".into(), "cool shorts".into()],
    )
    .await?;

    let _subscriber1 = Subscriber::connect(
        addrs.subscriptions,
        vec!["cool shorts".into(), "history".into()],
    )
    .await?;

    let mut publisher = publisher::PublisherClient::new(
        client::Config::default(),
        tcp::connect(addrs.publisher, Json::default()).await?,
    )
    .spawn()?;

    publisher
        .publish(context::current(), "calculus".into(), "sqrt(2)".into())
        .await?;

    publisher
        .publish(
            context::current(),
            "cool shorts".into(),
            "hello to all".into(),
        )
        .await?;

    publisher
        .publish(context::current(), "history".into(), "napoleon".to_string())
        .await?;

    info!("done.");

    Ok(())
}
<commit_msg>PubSub example needs to populate the subscription topics.<commit_after>// Copyright 2018 Google LLC
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT.

/// - The PubSub server sets up TCP listeners on 2 ports, the "subscriber" port and the "publisher"
///   port. Because both publishers and subscribers initiate their connections to the PubSub
///   server, the server requires no prior knowledge of either publishers or subscribers.
///
/// - Subscribers connect to the server on the server's "subscriber" port. Once a connection is
///   established, the server acts as the client of the Subscriber service, initially requesting
///   the topics the subscriber is interested in, and subsequently sending topical messages to the
///   subscriber.
///
/// - Publishers connect to the server on the "publisher" port and, once connected, they send
///   toopical messages via Publisher service to the server. The server then broadcasts each
///   messages to all clients subscribed to the topic of that message.
///
///       Subscriber                        Publisher                       PubSub Server
/// T1        |                                 |                                 |
/// T2        |-----Connect------------------------------------------------------>|
/// T3        |                                 |                                 |
/// T2        |<-------------------------------------------------------Topics-----|
/// T2        |-----(OK) Topics-------------------------------------------------->|
/// T3        |                                 |                                 |
/// T4        |                                 |-----Connect-------------------->|
/// T5        |                                 |                                 |
/// T6        |                                 |-----Publish-------------------->|
/// T7        |                                 |                                 |
/// T8        |<------------------------------------------------------Receive-----|
/// T9        |-----(OK) Receive------------------------------------------------->|
/// T10       |                                 |                                 |
/// T11       |                                 |<--------------(OK) Publish------|

use anyhow::anyhow;
use futures::{
    channel::oneshot,
    future::{self, AbortHandle},
    prelude::*,
};
use log::info;
use publisher::Publisher as _;
use std::{
    collections::HashMap,
    io,
    net::SocketAddr,
    sync::{Arc, Mutex, RwLock},
};
use subscriber::Subscriber as _;
use tarpc::{
    client, context,
    serde_transport::tcp,
    server::{self, Channel},
};
use tokio::net::ToSocketAddrs;
use tokio_serde::formats::Json;

pub mod subscriber {
    #[tarpc::service]
    pub trait Subscriber {
        async fn topics() -> Vec<String>;
        async fn receive(topic: String, message: String);
    }
}

pub mod publisher {
    #[tarpc::service]
    pub trait Publisher {
        async fn publish(topic: String, message: String);
    }
}

#[derive(Clone, Debug)]
struct Subscriber {
    local_addr: SocketAddr,
    topics: Vec<String>,
}

#[tarpc::server]
impl subscriber::Subscriber for Subscriber {
    async fn topics(self, _: context::Context) -> Vec<String> {
        self.topics.clone()
    }

    async fn receive(self, _: context::Context, topic: String, message: String) {
        info!(
            "[{}] received message on topic '{}': {}",
            self.local_addr, topic, message
        );
    }
}

struct SubscriberHandle(AbortHandle);

impl Drop for SubscriberHandle {
    fn drop(&mut self) {
        self.0.abort();
    }
}

impl Subscriber {
    async fn connect(
        publisher_addr: impl ToSocketAddrs,
        topics: Vec<String>,
    ) -> anyhow::Result<SubscriberHandle> {
        let publisher = tcp::connect(publisher_addr, Json::default()).await?;
        let local_addr = publisher.local_addr()?;
        let mut handler = server::BaseChannel::with_defaults(publisher)
            .respond_with(Subscriber { local_addr, topics }.serve());
        // The first request is for the topics being subscriibed to.
        match handler.next().await {
            Some(init_topics) => init_topics?.await,
            None => {
                return Err(anyhow!(
                    "[{}] Server never initialized the subscriber.",
                    local_addr
                ))
            }
        };
        let (handler, abort_handle) = future::abortable(handler.execute());
        tokio::spawn(async move {
            match handler.await {
                Ok(()) | Err(future::Aborted) => info!("[{}] subscriber shutdown.", local_addr),
            }
        });
        Ok(SubscriberHandle(abort_handle))
    }
}

#[derive(Debug)]
struct Subscription {
    subscriber: subscriber::SubscriberClient,
    topics: Vec<String>,
}

#[derive(Clone, Debug)]
struct Publisher {
    clients: Arc<Mutex<HashMap<SocketAddr, Subscription>>>,
    subscriptions: Arc<RwLock<HashMap<String, HashMap<SocketAddr, subscriber::SubscriberClient>>>>,
}

struct PublisherAddrs {
    publisher: SocketAddr,
    subscriptions: SocketAddr,
}

impl Publisher {
    async fn start(self) -> io::Result<PublisherAddrs> {
        let mut connecting_publishers = tcp::listen("localhost:0", Json::default).await?;

        let publisher_addrs = PublisherAddrs {
            publisher: connecting_publishers.local_addr(),
            subscriptions: self.clone().start_subscription_manager().await?,
        };

        info!("[{}] listening for publishers.", publisher_addrs.publisher);
        tokio::spawn(async move {
            // Because this is just an example, we know there will only be one publisher. In more
            // realistic code, this would be a loop to continually accept new publisher
            // connections.
            let publisher = connecting_publishers.next().await.unwrap().unwrap();
            info!("[{}] publisher connected.", publisher.peer_addr().unwrap());

            server::BaseChannel::with_defaults(publisher)
                .respond_with(self.serve())
                .execute()
                .await
        });

        Ok(publisher_addrs)
    }

    async fn start_subscription_manager(self) -> io::Result<SocketAddr> {
        let mut connecting_subscribers = tcp::listen("localhost:0", Json::default)
            .await?
            .filter_map(|r| future::ready(r.ok()));
        let new_subscriber_addr = connecting_subscribers.get_ref().local_addr();
        info!("[{}] listening for subscribers.", new_subscriber_addr);

        tokio::spawn(async move {
            while let Some(conn) = connecting_subscribers.next().await {
                let subscriber_addr = conn.peer_addr().unwrap();

                let tarpc::client::NewClient {
                    client: mut subscriber,
                    dispatch,
                } = subscriber::SubscriberClient::new(client::Config::default(), conn);
                let (ready_tx, ready) = oneshot::channel();
                self.clone()
                    .start_subscriber_gc(subscriber_addr, dispatch, ready);

                // Populate the topics
                if let Ok(topics) = subscriber.topics(context::current()).await {
                    self.clients.lock().unwrap().insert(
                        subscriber_addr,
                        Subscription {
                            subscriber: subscriber.clone(),
                            topics: topics.clone(),
                        },
                    );

                    info!("[{}] subscribed to topics: {:?}", subscriber_addr, topics);
                    let mut subscriptions = self.subscriptions.write().unwrap();
                    for topic in topics {
                        subscriptions
                            .entry(topic)
                            .or_insert_with(HashMap::new)
                            .insert(subscriber_addr, subscriber.clone());
                    }
                }
                // Signal that initialization is done.
                ready_tx.send(()).unwrap();
            }
        });

        Ok(new_subscriber_addr)
    }

    fn start_subscriber_gc(
        self,
        subscriber_addr: SocketAddr,
        client_dispatch: impl Future<Output = anyhow::Result<()>> + Send + 'static,
        subscriber_ready: oneshot::Receiver<()>,
    ) {
        tokio::spawn(async move {
            if let Err(e) = client_dispatch.await {
                info!("[{}] subscriber connection broken: {:?}", subscriber_addr, e)
            }
            // Don't clean up the subscriber until initialization is done.
            let _ = subscriber_ready.await;
            if let Some(subscription) = self.clients.lock().unwrap().remove(&subscriber_addr) {
                info!("[{} unsubscribing from topics: {:?}", subscriber_addr, subscription.topics);
                let mut subscriptions = self.subscriptions.write().unwrap();
                for topic in subscription.topics {
                    let subscribers = subscriptions.get_mut(&topic).unwrap();
                    subscribers.remove(&subscriber_addr);
                    if subscribers.is_empty() {
                        subscriptions.remove(&topic);
                    }
                }
            }
        });
    }
}

#[tarpc::server]
impl publisher::Publisher for Publisher {
    async fn publish(self, _: context::Context, topic: String, message: String) {
        info!("received message to publish.");
        let mut subscribers = match self.subscriptions.read().unwrap().get(&topic) {
            None => return,
            Some(subscriptions) => subscriptions.clone(),
        };
        let mut publications = Vec::new();
        for client in subscribers.values_mut() {
            publications.push(client.receive(context::current(), topic.clone(), message.clone()));
        }
        // Ignore failing subscribers. In a real pubsub, you'd want to continually retry until
        // subscribers ack. Of course, a lot would be different in a real pubsub :)
        for response in future::join_all(publications).await {
            if let Err(e) = response {
                info!("failed to broadcast to subscriber: {}", e);
            }
        }
    }
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    env_logger::init();

    let clients = Arc::new(Mutex::new(HashMap::new()));
    let addrs = Publisher {
        clients,
        subscriptions: Arc::new(RwLock::new(HashMap::new())),
    }
    .start()
    .await?;

    let _subscriber0 = Subscriber::connect(
        addrs.subscriptions,
        vec!["calculus".into(), "cool shorts".into()],
    )
    .await?;

    let _subscriber1 = Subscriber::connect(
        addrs.subscriptions,
        vec!["cool shorts".into(), "history".into()],
    )
    .await?;

    let mut publisher = publisher::PublisherClient::new(
        client::Config::default(),
        tcp::connect(addrs.publisher, Json::default()).await?,
    )
    .spawn()?;

    publisher
        .publish(context::current(), "calculus".into(), "sqrt(2)".into())
        .await?;

    publisher
        .publish(
            context::current(),
            "cool shorts".into(),
            "hello to all".into(),
        )
        .await?;

    publisher
        .publish(context::current(), "history".into(), "napoleon".to_string())
        .await?;

    drop(_subscriber0);

    publisher
        .publish(
            context::current(),
            "cool shorts".into(),
            "hello to who?".into(),
        )
        .await?;

    info!("done.");

    Ok(())
}
<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

// static USAGE: &'static str = "
// Usage: pagerank <source> <workers>
// ";

static USAGE: &'static str = "
Usage: scc <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; nodes / peers as usize];    // local rank accumulation
        let mut dst = vec![0.0; nodes];                     // local rank accumulation

        // for node in 0..graph.nodes() {
        //     for &b in graph.edges(node) {
        //         dst[b as usize] = 1.0;
        //     }
        // }
        //
        // let mut count = 0;
        // for x in &dst { if *x != 0.0 { count += 1; } }
        // println!("can reach {} elements of {}", count, dst.len());
        //
        // for node in 0..graph.nodes() {
        //     if node % peers == index {
        //         for &b in graph.edges(node) {
        //             dst[b as usize] = 1.0;
        //         }
        //     }
        // }
        //
        // let mut count = 0;
        // for x in &dst { if *x != 0.0 { count += 1; } }
        // println!("can reach {} elements of {} locally", count, dst.len());
        //
        // let mut dst = vec![0.0; nodes];                     // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/
                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(nodes/workers) { acc.push(0.0); }
        //             assert!(acc.len() == nodes/workers);
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>moar diagnostics<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; nodes / peers as usize];    // local rank accumulation
        let mut dst = vec![0.0; nodes];                     // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    let mut counter = 0;
                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                            counter += 1;
                        }
                    }
                    // \------ end familiar part ------/
                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("iteration {:?}: {}s; processed {} edges", iter, time::precise_time_s() - start, counter);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(nodes/workers) { acc.push(0.0); }
        //             assert!(acc.len() == nodes/workers);
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>


<commit_before>#![feature(phase)]
#![crate_name = "cube"]

extern crate cgmath;
extern crate gfx;
#[phase(plugin)]
extern crate gfx_macros;
extern crate glfw;
extern crate native;
extern crate time;

use cgmath::matrix::{Matrix, Matrix4};
use cgmath::point::Point3;
use cgmath::transform::{Transform, AffineMatrix3};
use cgmath::vector::Vector3;
use gfx::{Device, DeviceHelper};
use glfw::Context;

//----------------------------------------
// Cube associated data

#[vertex_format]
struct Vertex {
    #[as_float]
    a_Pos: [i8, ..3],
    #[as_float]
    a_TexCoord: [u8, ..2],
}

impl Vertex {
    fn new(pos: [i8, ..3], tc: [u8, ..2]) -> Vertex {
        Vertex {
            a_Pos: pos,
            a_TexCoord: tc,
        }
    }
}

#[shader_param(Program)]
struct Params {
    u_ModelViewProj: [[f32, ..4], ..4],
    t_Color: gfx::shade::TextureParam,
}

static VERTEX_SRC: gfx::ShaderSource = shaders! {
GLSL_120: b"
    #version 120
    attribute vec3 a_Pos;
    attribute vec2 a_TexCoord;
    varying vec2 v_TexCoord;
    uniform mat4 u_ModelViewProj;
    void main() {
        v_TexCoord = a_TexCoord;
        gl_Position = u_ModelViewProj * vec4(a_Pos, 1.0);
    }
"
GLSL_150: b"
    #version 150 core
    in vec3 a_Pos;
    in vec2 a_TexCoord;
    out vec2 v_TexCoord;
    uniform mat4 u_ModelViewProj;
    void main() {
        v_TexCoord = a_TexCoord;
        gl_Position = u_ModelViewProj * vec4(a_Pos, 1.0);
    }
"
};

static FRAGMENT_SRC: gfx::ShaderSource = shaders! {
GLSL_120: b"
    #version 120
    varying vec2 v_TexCoord;
    uniform sampler2D t_Color;
    void main() {
        vec4 tex = texture2D(t_Color, v_TexCoord);
        float blend = dot(v_TexCoord-vec2(0.5,0.5), v_TexCoord-vec2(0.5,0.5));
        gl_FragColor = mix(tex, vec4(0.0,0.0,0.0,0.0), blend*1.0);
    }
"
GLSL_150: b"
    #version 150 core
    in vec2 v_TexCoord;
    out vec4 o_Color;
    uniform sampler2D t_Color;
    void main() {
        vec4 tex = texture(t_Color, v_TexCoord);
        float blend = dot(v_TexCoord-vec2(0.5,0.5), v_TexCoord-vec2(0.5,0.5));
        o_Color = mix(tex, vec4(0.0,0.0,0.0,0.0), blend*1.0);
    }
"
};

//----------------------------------------

// We need to run on the main thread, so ensure we are using the `native` runtime. This is
// technically not needed, since this is the default, but it's not guaranteed.
#[start]
fn start(argc: int, argv: *const *const u8) -> int {
     native::start(argc, argv, main)
}

fn main() {
    let glfw = glfw::init(glfw::FAIL_ON_ERRORS).unwrap();

    glfw.window_hint(glfw::ContextVersion(3, 2));
    glfw.window_hint(glfw::OpenglForwardCompat(true));
    glfw.window_hint(glfw::OpenglProfile(glfw::OpenGlCoreProfile));

    let (window, events) = glfw
        .create_window(640, 480, "Cube example", glfw::Windowed)
        .expect("Failed to create GLFW window.");

    window.make_current();
    glfw.set_error_callback(glfw::FAIL_ON_ERRORS);
    window.set_key_polling(true); // so we can quit when Esc is pressed
    let (w, h) = window.get_framebuffer_size();
    let frame = gfx::Frame::new(w as u16, h as u16);

    let mut device = gfx::GlDevice::new(|s| glfw.get_proc_address(s));
    let mut list = device.create_draw_list();

    let state = gfx::DrawState::new().depth(gfx::state::LessEqual, true);

    let vertex_data = vec![
        //top (0, 0, 1)
        Vertex::new([-1, -1,  1], [0, 0]),
        Vertex::new([ 1, -1,  1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([-1,  1,  1], [0, 1]),
        //bottom (0, 0, -1)
        Vertex::new([ 1,  1, -1], [0, 0]),
        Vertex::new([-1,  1, -1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([ 1, -1, -1], [0, 1]),
        //right (1, 0, 0)
        Vertex::new([ 1, -1, -1], [0, 0]),
        Vertex::new([ 1,  1, -1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([ 1, -1,  1], [0, 1]),
        //left (-1, 0, 0)
        Vertex::new([-1,  1,  1], [0, 0]),
        Vertex::new([-1, -1,  1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([-1,  1, -1], [0, 1]),
        //front (0, 1, 0)
        Vertex::new([-1,  1, -1], [0, 0]),
        Vertex::new([ 1,  1, -1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([-1,  1,  1], [0, 1]),
        //back (0, -1, 0)
        Vertex::new([ 1, -1,  1], [0, 0]),
        Vertex::new([-1, -1,  1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([ 1, -1, -1], [0, 1]),
    ];

    let mesh = device.create_mesh(vertex_data);

    let slice = {
        let index_data = vec![
            0u8, 1, 2, 2, 3, 0,    //top
            4, 5, 6, 6, 7, 4,       //bottom
            8, 9, 10, 10, 11, 8,    //right
            12, 13, 14, 14, 16, 12, //left
            16, 17, 18, 18, 19, 16, //front
            20, 21, 22, 22, 23, 20, //back
        ];

        let buf = device.create_buffer_static(&index_data);
        gfx::IndexSlice8(buf, 0, 36)
    };

    let tinfo = gfx::tex::TextureInfo {
        width: 1,
        height: 1,
        depth: 1,
        mipmap_range: (0, 1),
        kind: gfx::tex::Texture2D,
        format: gfx::tex::RGBA8,
    };
    let img_info = tinfo.to_image_info();
    let texture = device.create_texture(tinfo).unwrap();
    device.update_texture(&texture, &img_info,
                          &vec![0x20u8, 0xA0u8, 0xC0u8, 0x00u8])
           .unwrap();

    let sampler = device.create_sampler(gfx::tex::SamplerInfo::new(
        gfx::tex::Bilinear, gfx::tex::Clamp));

    let mut prog = {
        let data = Params {
            u_ModelViewProj: [
                [1.0, 0.0, 0.0, 0.0],
                [0.0, 1.0, 0.0, 0.0],
                [0.0, 0.0, 1.0, 0.0],
                [0.0, 0.0, 0.0, 1.0],
            ],
            t_Color: (texture, Some(sampler)),
        };
        device.link_program(data, VERTEX_SRC.clone(), FRAGMENT_SRC.clone())
               .unwrap()
    };

    let mut m_model = Matrix4::<f32>::identity();
    let m_viewproj = {
        let mv: AffineMatrix3<f32> = Transform::look_at(
            &Point3::new(1.5f32, -5.0, 3.0),
            &Point3::new(0f32, 0.0, 0.0),
            &Vector3::unit_z()
            );
        let aspect = w as f32 / h as f32;
        let mp = cgmath::projection::perspective(
            cgmath::angle::deg(45f32), aspect, 1f32, 10f32);
        mp.mul_m(&mv.mat)
    };

    while !window.should_close() {
        glfw.poll_events();
        // quit when Esc is pressed.
        for (_, event) in glfw::flush_messages(&events) {
            match event {
                glfw::KeyEvent(glfw::KeyEscape, _, glfw::Press, _) =>
                    window.set_should_close(true),
                _ => {},
            }
        }
        // render
        list.reset();
        list.clear(
            gfx::ClearData {
                color: Some(gfx::Color([0.3, 0.3, 0.3, 1.0])),
                depth: Some(1.0),
                stencil: None,
            },
            &frame
        );
        m_model.x.x = 1.0;
        prog.data.u_ModelViewProj = {
            let m = m_viewproj.mul_m(&m_model);
            [ //TODO: add raw convertion methods to cgmath
                [m.x.x, m.x.y, m.x.z, m.x.w],
                [m.y.x, m.y.y, m.y.z, m.y.w],
                [m.z.x, m.z.y, m.z.z, m.z.w],
                [m.w.x, m.w.y, m.w.z, m.w.w]
            ]
        };
        list.draw(&mesh, slice, &frame, &prog, &state)
            .unwrap();
        device.submit(list.as_slice());
        window.swap_buffers();
    }
}
<commit_msg>Clean up cgmath stuff<commit_after>#![feature(phase)]
#![crate_name = "cube"]

extern crate cgmath;
extern crate gfx;
#[phase(plugin)]
extern crate gfx_macros;
extern crate glfw;
extern crate native;
extern crate time;

use cgmath::angle;
use cgmath::array::FixedArray;
use cgmath::matrix::{Matrix, Matrix4};
use cgmath::point::Point3;
use cgmath::projection;
use cgmath::transform::{Transform, AffineMatrix3};
use cgmath::vector::Vector3;
use gfx::{Device, DeviceHelper};
use glfw::Context;

//----------------------------------------
// Cube associated data

#[vertex_format]
struct Vertex {
    #[as_float]
    a_Pos: [i8, ..3],
    #[as_float]
    a_TexCoord: [u8, ..2],
}

impl Vertex {
    fn new(pos: [i8, ..3], tc: [u8, ..2]) -> Vertex {
        Vertex {
            a_Pos: pos,
            a_TexCoord: tc,
        }
    }
}

#[shader_param(Program)]
struct Params {
    u_ModelViewProj: [[f32, ..4], ..4],
    t_Color: gfx::shade::TextureParam,
}

static VERTEX_SRC: gfx::ShaderSource = shaders! {
GLSL_120: b"
    #version 120
    attribute vec3 a_Pos;
    attribute vec2 a_TexCoord;
    varying vec2 v_TexCoord;
    uniform mat4 u_ModelViewProj;
    void main() {
        v_TexCoord = a_TexCoord;
        gl_Position = u_ModelViewProj * vec4(a_Pos, 1.0);
    }
"
GLSL_150: b"
    #version 150 core
    in vec3 a_Pos;
    in vec2 a_TexCoord;
    out vec2 v_TexCoord;
    uniform mat4 u_ModelViewProj;
    void main() {
        v_TexCoord = a_TexCoord;
        gl_Position = u_ModelViewProj * vec4(a_Pos, 1.0);
    }
"
};

static FRAGMENT_SRC: gfx::ShaderSource = shaders! {
GLSL_120: b"
    #version 120
    varying vec2 v_TexCoord;
    uniform sampler2D t_Color;
    void main() {
        vec4 tex = texture2D(t_Color, v_TexCoord);
        float blend = dot(v_TexCoord-vec2(0.5,0.5), v_TexCoord-vec2(0.5,0.5));
        gl_FragColor = mix(tex, vec4(0.0,0.0,0.0,0.0), blend*1.0);
    }
"
GLSL_150: b"
    #version 150 core
    in vec2 v_TexCoord;
    out vec4 o_Color;
    uniform sampler2D t_Color;
    void main() {
        vec4 tex = texture(t_Color, v_TexCoord);
        float blend = dot(v_TexCoord-vec2(0.5,0.5), v_TexCoord-vec2(0.5,0.5));
        o_Color = mix(tex, vec4(0.0,0.0,0.0,0.0), blend*1.0);
    }
"
};

//----------------------------------------

// We need to run on the main thread, so ensure we are using the `native` runtime. This is
// technically not needed, since this is the default, but it's not guaranteed.
#[start]
fn start(argc: int, argv: *const *const u8) -> int {
     native::start(argc, argv, main)
}

fn main() {
    let glfw = glfw::init(glfw::FAIL_ON_ERRORS).unwrap();

    glfw.window_hint(glfw::ContextVersion(3, 2));
    glfw.window_hint(glfw::OpenglForwardCompat(true));
    glfw.window_hint(glfw::OpenglProfile(glfw::OpenGlCoreProfile));

    let (window, events) = glfw
        .create_window(640, 480, "Cube example", glfw::Windowed)
        .expect("Failed to create GLFW window.");

    window.make_current();
    glfw.set_error_callback(glfw::FAIL_ON_ERRORS);
    window.set_key_polling(true); // so we can quit when Esc is pressed
    let (w, h) = window.get_framebuffer_size();
    let frame = gfx::Frame::new(w as u16, h as u16);

    let mut device = gfx::GlDevice::new(|s| glfw.get_proc_address(s));
    let mut list = device.create_draw_list();

    let state = gfx::DrawState::new().depth(gfx::state::LessEqual, true);

    let vertex_data = vec![
        //top (0, 0, 1)
        Vertex::new([-1, -1,  1], [0, 0]),
        Vertex::new([ 1, -1,  1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([-1,  1,  1], [0, 1]),
        //bottom (0, 0, -1)
        Vertex::new([ 1,  1, -1], [0, 0]),
        Vertex::new([-1,  1, -1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([ 1, -1, -1], [0, 1]),
        //right (1, 0, 0)
        Vertex::new([ 1, -1, -1], [0, 0]),
        Vertex::new([ 1,  1, -1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([ 1, -1,  1], [0, 1]),
        //left (-1, 0, 0)
        Vertex::new([-1,  1,  1], [0, 0]),
        Vertex::new([-1, -1,  1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([-1,  1, -1], [0, 1]),
        //front (0, 1, 0)
        Vertex::new([-1,  1, -1], [0, 0]),
        Vertex::new([ 1,  1, -1], [1, 0]),
        Vertex::new([ 1,  1,  1], [1, 1]),
        Vertex::new([-1,  1,  1], [0, 1]),
        //back (0, -1, 0)
        Vertex::new([ 1, -1,  1], [0, 0]),
        Vertex::new([-1, -1,  1], [1, 0]),
        Vertex::new([-1, -1, -1], [1, 1]),
        Vertex::new([ 1, -1, -1], [0, 1]),
    ];

    let mesh = device.create_mesh(vertex_data);

    let slice = {
        let index_data = vec![
            0u8, 1, 2, 2, 3, 0,    //top
            4, 5, 6, 6, 7, 4,       //bottom
            8, 9, 10, 10, 11, 8,    //right
            12, 13, 14, 14, 16, 12, //left
            16, 17, 18, 18, 19, 16, //front
            20, 21, 22, 22, 23, 20, //back
        ];

        let buf = device.create_buffer_static(&index_data);
        gfx::IndexSlice8(buf, 0, 36)
    };

    let tinfo = gfx::tex::TextureInfo {
        width: 1,
        height: 1,
        depth: 1,
        mipmap_range: (0, 1),
        kind: gfx::tex::Texture2D,
        format: gfx::tex::RGBA8,
    };
    let img_info = tinfo.to_image_info();
    let texture = device.create_texture(tinfo).unwrap();
    device.update_texture(&texture, &img_info,
                          &vec![0x20u8, 0xA0u8, 0xC0u8, 0x00u8])
           .unwrap();

    let sampler = device.create_sampler(gfx::tex::SamplerInfo::new(
        gfx::tex::Bilinear, gfx::tex::Clamp));

    let mut prog = {
        let data = Params {
            u_ModelViewProj: Matrix4::identity().into_fixed(),
            t_Color: (texture, Some(sampler)),
        };
        device.link_program(data, VERTEX_SRC.clone(), FRAGMENT_SRC.clone())
               .unwrap()
    };

    let mut m_model = Matrix4::<f32>::identity();
    let m_viewproj = {
        let mv: AffineMatrix3<f32> = Transform::look_at(
            &Point3::new(1.5f32, -5.0, 3.0),
            &Point3::new(0f32, 0.0, 0.0),
            &Vector3::unit_z()
        );
        let aspect = w as f32 / h as f32;
        let mp = projection::perspective(angle::deg(45f32), aspect,
                                         1f32, 10f32);
        mp.mul_m(&mv.mat)
    };

    while !window.should_close() {
        glfw.poll_events();
        // quit when Esc is pressed.
        for (_, event) in glfw::flush_messages(&events) {
            match event {
                glfw::KeyEvent(glfw::KeyEscape, _, glfw::Press, _) =>
                    window.set_should_close(true),
                _ => {},
            }
        }
        // render
        list.reset();
        list.clear(
            gfx::ClearData {
                color: Some(gfx::Color([0.3, 0.3, 0.3, 1.0])),
                depth: Some(1.0),
                stencil: None,
            },
            &frame
        );
        m_model.x.x = 1.0;
        prog.data.u_ModelViewProj = m_viewproj.mul_m(&m_model).into_fixed();
        list.draw(&mesh, slice, &frame, &prog, &state).unwrap();
        device.submit(list.as_slice());
        window.swap_buffers();
    }
}
<|endoftext|>


<commit_before>use sodiumoxide::crypto::sign::Signature;
use crypto::{SigPair, PublicKey};
use std::io::{Read, Cursor};
use std::str::FromStr;
use asset::Asset;

trait DecodeTx<T> {
    /**
     * Returns the deserialized type T and the bytes read. Otherwise None is
     * returned if there's a decoding failure.
     */
    fn decode(c: &mut Cursor<Vec<u8>>) -> Option<(T, u16)>;
}

pub trait BufWrite {
    fn push_u16(&mut self, num: u16);
    fn push_u32(&mut self, num: u32);
    fn push_u64(&mut self, num: u64);
    fn push_bytes(&mut self, slice: &[u8]);
    fn push_pub_key(&mut self, key: &PublicKey);
    fn push_sig_pair(&mut self, pair: &SigPair);
    fn push_asset(&mut self, asset: &Asset);
}

impl BufWrite for Vec<u8> {
    fn push_u16(&mut self, mut num: u16) {
        num = num.to_be();
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_u32(&mut self, mut num: u32) {
        num = num.to_be();
        self.push((num >> 24) as u8);
        self.push((num >> 16) as u8);
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_u64(&mut self, mut num: u64) {
        num = num.to_be();
        self.push((num >> 56) as u8);
        self.push((num >> 48) as u8);
        self.push((num >> 40) as u8);
        self.push((num >> 32) as u8);
        self.push((num >> 24) as u8);
        self.push((num >> 16) as u8);
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_bytes(&mut self, other: &[u8]) {
        if other.is_empty() {
            self.push_u32(0);
            return
        }
        self.push_u32(other.len() as u32);
        self.extend_from_slice(other);
    }

    fn push_pub_key(&mut self, key: &PublicKey) {
        self.push_bytes(key.as_bytes());
    }

    fn push_sig_pair(&mut self, pair: &SigPair) {
        self.push_pub_key(&pair.pub_key);
        self.push_bytes(pair.signature.as_ref());
    }

    fn push_asset(&mut self, asset: &Asset) {
        self.push_bytes((&*asset.to_str()).as_ref());
    }
}

pub trait BufRead {
    fn take_u8(&mut self) -> Option<u8>;
    fn take_u16(&mut self) -> Option<u16>;
    fn take_u32(&mut self) -> Option<u32>;
    fn take_u64(&mut self) -> Option<u64>;
    fn take_bytes(&mut self) -> Option<Vec<u8>>;
    fn take_pub_key(&mut self) -> Option<PublicKey>;
    fn take_sig_pair(&mut self) -> Option<SigPair>;
    fn take_asset(&mut self) -> Option<Asset>;
}

impl<'a, T: AsRef<[u8]> + Read> BufRead for Cursor<T> {
    fn take_u8(&mut self) -> Option<u8> {
        let mut buf = [0u8;1];
        self.read_exact(&mut buf).ok()?;
        Some(buf[0])
    }

    fn take_u16(&mut self) -> Option<u16> {
        let mut buf = [0u8;2];
        self.read_exact(&mut buf).ok()?;
        Some(u16::from_be((u16::from(buf[0]) << 8) | u16::from(buf[1])))
    }

    fn take_u32(&mut self) -> Option<u32> {
        let mut buf = [0u8;4];
        self.read_exact(&mut buf).ok()?;
        Some(u32::from_be((u32::from(buf[0]) << 24)
                | (u32::from(buf[1]) << 16)
                | (u32::from(buf[2]) << 8)
                | (u32::from(buf[3]))))
    }

    fn take_u64(&mut self) -> Option<u64> {
        let mut buf = [0u8;8];
        self.read_exact(&mut buf).ok()?;
        Some(u64::from_be((u64::from(buf[0]) << 56)
                | (u64::from(buf[1]) << 48)
                | (u64::from(buf[2]) << 40)
                | (u64::from(buf[3]) << 32)
                | (u64::from(buf[4]) << 24)
                | (u64::from(buf[5]) << 16)
                | (u64::from(buf[6]) << 8)
                | u64::from(buf[7])))
    }

    fn take_bytes(&mut self) -> Option<Vec<u8>> {
        let len = self.take_u32()? as usize;
        let mut buf = Vec::with_capacity(len);
        unsafe { buf.set_len(len); }
        self.read_exact(&mut buf).ok()?;
        Some(buf)
    }

    fn take_pub_key(&mut self) -> Option<PublicKey> {
        let buf = self.take_bytes()?;
        PublicKey::from_bytes(&buf)
    }

    fn take_sig_pair(&mut self) -> Option<SigPair> {
        let pub_key = self.take_pub_key()?;
        let signature = Signature::from_slice(&self.take_bytes()?)?;
        Some(SigPair {
            pub_key,
            signature
        })
    }

    fn take_asset(&mut self) -> Option<Asset> {
        let bytes = self.take_bytes()?;
        let s = String::from_utf8(bytes).ok()?;
        Asset::from_str(&s).ok()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_u64_serialization() {
        let num: u64 = 0xFF000000_FFFFFFFF;
        let mut v = Vec::with_capacity(8);
        v.push_u64(num);

        let dec = Cursor::<&[u8]>::new(&v).take_u64().unwrap();
        assert_eq!(num, dec);
    }

    #[test]
    fn test_asset_serialization() {
        let a = Asset::from_str("12.34 GOLD").unwrap();
        let mut v = vec![];
        v.push_asset(&a);

        let mut c = Cursor::<&[u8]>::new(&v);
        let b = c.take_asset().unwrap();
        assert_eq!(a.to_str(), b.to_str());
    }
}
<commit_msg>Remove unused trait<commit_after>use sodiumoxide::crypto::sign::Signature;
use crypto::{SigPair, PublicKey};
use std::io::{Read, Cursor};
use std::str::FromStr;
use asset::Asset;

pub trait BufWrite {
    fn push_u16(&mut self, num: u16);
    fn push_u32(&mut self, num: u32);
    fn push_u64(&mut self, num: u64);
    fn push_bytes(&mut self, slice: &[u8]);
    fn push_pub_key(&mut self, key: &PublicKey);
    fn push_sig_pair(&mut self, pair: &SigPair);
    fn push_asset(&mut self, asset: &Asset);
}

impl BufWrite for Vec<u8> {
    fn push_u16(&mut self, mut num: u16) {
        num = num.to_be();
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_u32(&mut self, mut num: u32) {
        num = num.to_be();
        self.push((num >> 24) as u8);
        self.push((num >> 16) as u8);
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_u64(&mut self, mut num: u64) {
        num = num.to_be();
        self.push((num >> 56) as u8);
        self.push((num >> 48) as u8);
        self.push((num >> 40) as u8);
        self.push((num >> 32) as u8);
        self.push((num >> 24) as u8);
        self.push((num >> 16) as u8);
        self.push((num >> 8) as u8);
        self.push(num as u8);
    }

    fn push_bytes(&mut self, other: &[u8]) {
        if other.is_empty() {
            self.push_u32(0);
            return
        }
        self.push_u32(other.len() as u32);
        self.extend_from_slice(other);
    }

    fn push_pub_key(&mut self, key: &PublicKey) {
        self.push_bytes(key.as_bytes());
    }

    fn push_sig_pair(&mut self, pair: &SigPair) {
        self.push_pub_key(&pair.pub_key);
        self.push_bytes(pair.signature.as_ref());
    }

    fn push_asset(&mut self, asset: &Asset) {
        self.push_bytes((&*asset.to_str()).as_ref());
    }
}

pub trait BufRead {
    fn take_u8(&mut self) -> Option<u8>;
    fn take_u16(&mut self) -> Option<u16>;
    fn take_u32(&mut self) -> Option<u32>;
    fn take_u64(&mut self) -> Option<u64>;
    fn take_bytes(&mut self) -> Option<Vec<u8>>;
    fn take_pub_key(&mut self) -> Option<PublicKey>;
    fn take_sig_pair(&mut self) -> Option<SigPair>;
    fn take_asset(&mut self) -> Option<Asset>;
}

impl<'a, T: AsRef<[u8]> + Read> BufRead for Cursor<T> {
    fn take_u8(&mut self) -> Option<u8> {
        let mut buf = [0u8;1];
        self.read_exact(&mut buf).ok()?;
        Some(buf[0])
    }

    fn take_u16(&mut self) -> Option<u16> {
        let mut buf = [0u8;2];
        self.read_exact(&mut buf).ok()?;
        Some(u16::from_be((u16::from(buf[0]) << 8) | u16::from(buf[1])))
    }

    fn take_u32(&mut self) -> Option<u32> {
        let mut buf = [0u8;4];
        self.read_exact(&mut buf).ok()?;
        Some(u32::from_be((u32::from(buf[0]) << 24)
                | (u32::from(buf[1]) << 16)
                | (u32::from(buf[2]) << 8)
                | (u32::from(buf[3]))))
    }

    fn take_u64(&mut self) -> Option<u64> {
        let mut buf = [0u8;8];
        self.read_exact(&mut buf).ok()?;
        Some(u64::from_be((u64::from(buf[0]) << 56)
                | (u64::from(buf[1]) << 48)
                | (u64::from(buf[2]) << 40)
                | (u64::from(buf[3]) << 32)
                | (u64::from(buf[4]) << 24)
                | (u64::from(buf[5]) << 16)
                | (u64::from(buf[6]) << 8)
                | u64::from(buf[7])))
    }

    fn take_bytes(&mut self) -> Option<Vec<u8>> {
        let len = self.take_u32()? as usize;
        let mut buf = Vec::with_capacity(len);
        unsafe { buf.set_len(len); }
        self.read_exact(&mut buf).ok()?;
        Some(buf)
    }

    fn take_pub_key(&mut self) -> Option<PublicKey> {
        let buf = self.take_bytes()?;
        PublicKey::from_bytes(&buf)
    }

    fn take_sig_pair(&mut self) -> Option<SigPair> {
        let pub_key = self.take_pub_key()?;
        let signature = Signature::from_slice(&self.take_bytes()?)?;
        Some(SigPair {
            pub_key,
            signature
        })
    }

    fn take_asset(&mut self) -> Option<Asset> {
        let bytes = self.take_bytes()?;
        let s = String::from_utf8(bytes).ok()?;
        Asset::from_str(&s).ok()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_u64_serialization() {
        let num: u64 = 0xFF000000_FFFFFFFF;
        let mut v = Vec::with_capacity(8);
        v.push_u64(num);

        let dec = Cursor::<&[u8]>::new(&v).take_u64().unwrap();
        assert_eq!(num, dec);
    }

    #[test]
    fn test_asset_serialization() {
        let a = Asset::from_str("12.34 GOLD").unwrap();
        let mut v = vec![];
        v.push_asset(&a);

        let mut c = Cursor::<&[u8]>::new(&v);
        let b = c.take_asset().unwrap();
        assert_eq!(a.to_str(), b.to_str());
    }
}
<|endoftext|>


<commit_before>fn main() {
    println!("Hello, world!");
}
<commit_msg>small changes<commit_after>// Open a file, or read a file on stdin,
// and print it to stdout with line numbers
use std::io;
use std::io::prelude::*;
use std::fs::File;

fn main() {
    let lines = "hello\nworld".lines();

    for (linenumber, line) in lines.enumerate() {
        println!("{}: {}", linenumber, line);
    }
}
<|endoftext|>


<commit_before>use {AnyUri, Result, Error, ErrorKind, v1_4, v1_5};
use std::fmt::{self, Display, Formatter};
use std::io::Read;
use std::str::FromStr;
use xml::attribute::OwnedAttribute;
use xml::common::Position;
use xml::name::OwnedName;
use xml::namespace::Namespace;
use xml::reader::{EventReader, ParserConfig};
use xml::reader::XmlEvent::*;

pub static PARSER_CONFIG: ParserConfig = ParserConfig {
    trim_whitespace: true,
    whitespace_to_characters: true,
    cdata_to_characters: true,
    ignore_comments: true,
    coalesce_characters: true,
};

#[derive(Debug)]
struct ElementStart {
    name: OwnedName,
    attributes: Vec<OwnedAttribute>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChildOccurrences {
    Optional,
    Required,
    Many,
}

pub struct ElementConfiguration<'a, R: 'a + Read> {
    pub name: &'static str,
    pub children: &'a mut [ChildConfiguration<'a, R>],
}

impl<'a, R: 'a + Read> ElementConfiguration<'a, R> {
    pub fn parse(mut self, reader: &mut EventReader<R>) -> Result<()> {
        // Keep track of the text position for the root element so that it can be used for error
        // messages.
        let root_position = reader.position();
        let mut current_child = 0;

        'elements: while let Some(element) = start_element(reader, self.name)? {
            for child_index in current_child..self.children.len() {
                let child_name = {
                    let child = &mut self.children[child_index];

                    if child.name == element.name.local_name {
                        // We've found a valid child, hooray!
                        (child.action)(reader, element.attributes)?;

                        // Either advance current_child or don't, depending on if it's allowed to repeat.
                        if child.occurrences != ChildOccurrences::Many {
                            current_child = child_index + 1;
                        }

                        continue 'elements;
                    } else if child.occurrences != ChildOccurrences::Required {
                        current_child = child_index + 1;
                        continue;
                    }

                    child.name
                };

                // If the child we found was a valid child (just in the wrong spot) then we
                // want to return a `MissingElement` error, but if the child we found was
                // not a valid child, then we want to return an `UnexpectedElement` error.
                if self.is_valid_child(&*element.name.local_name) {
                    // We skipped a required child and that is no good and also very bad.
                    return Err(Error {
                        position: root_position,
                        kind: ErrorKind::MissingElement {
                            parent: self.name,
                            expected: child_name,
                        },
                    });
                } else {
                    return Err(Error {
                        position: reader.position(),
                        kind: ErrorKind::UnexpectedElement {
                            parent: self.name,
                            element: element.name.local_name,
                            expected: self.collect_expected_children(),
                        },
                    });
                }
            }

            // Child didn't appear in list of children, error o'clock.
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: self.name,
                    element: element.name.local_name,
                    expected: self.collect_expected_children(),
                },
            })
        }

        // Verify that there are no remaining required children.
        for child in &self.children[current_child..] {
            if child.occurrences == ChildOccurrences::Required {
                return Err(Error {
                    position: root_position,
                    kind: ErrorKind::MissingElement {
                        parent: self.name,
                        expected: child.name,
                    },
                });
            }
        }

        Ok(())
    }

    fn collect_expected_children(&self) -> Vec<&'static str> {
        let mut names = Vec::with_capacity(self.children.len());
        for child in self.children.iter() {
            names.push(child.name);
        }
        names
    }

    fn is_valid_child(&self, name: &str) -> bool {
        for child in self.children.iter() {
            if child.name == name {
                return true;
            }
        }

        false
    }
}

pub struct ChildConfiguration<'a, R: 'a + Read> {
    pub name: &'static str,
    pub occurrences: ChildOccurrences,
    pub action: &'a mut FnMut(&mut EventReader<R>, Vec<OwnedAttribute>) -> Result<()>,
}

pub fn parse<R: Read>(mut reader: EventReader<R>) -> Result<v1_5::Collada> {
    pub static COLLADA_ATTRIBS: &'static [&'static str] = &["version", "xmlns", "base"];

    // Eat the `StartDocument` event. It has no useful information for our purposes, but it
    // will always be the first event emitted, even if there's no XML declaration at the
    // beginning of the document. This is defined as part of the xml-rs API as of v0.3.5,
    // but it's possible this can will change in the future.
    match reader.next()? {
        StartDocument { .. } => {},
        _ => panic!("First event from EventReader wasn't StartDocument"),
    }

    // The next element will always be the `<COLLADA>` tag. This will specify what version of
    // the COLLADA spec is being used, which is how we'll determine our sub-parser.
    let attributes = match reader.next()? {
        StartElement { name, attributes, namespace: _ } => {
            // If the element isn't the `<COLLADA>` tag then the document is malformed,
            // return an error.
            if name.local_name != "COLLADA" {
                return Err(Error {
                    position: reader.position(),
                    kind: ErrorKind::UnexpectedRootElement {
                        element: name.local_name,
                    }
                })
            }

            attributes
        }

        // I'm *almost* 100% certain that the only event that can follow the `StartDocument`
        // event is a `StartElement` event. As of v0.3.5, xml-rs doesn't support
        // `<!DOCTYPE>` or processing instructions, and it ignores whitespace and comments
        // (according to how we configure the parser), and those are the only things allowed
        // between `StartDocument` and the first `StartElement`. If xml-rs changes its
        // behavior this will need to be updated.
        event @ _ => { panic!("Unexpected event: {:?}", event); }
    };

    // Valide the attributes on the `<COLLADA>` tag.
    // Use boolean flags to track if specific attributes have been encountered.
    let mut version = None;
    let mut base_uri = None;

    for attribute in attributes {
        // NOTE: I'm using `if` blocks instead of `match` here because using `match`
        // won't allow for the name to be moved out of `attribute`. Using `if` saves
        // some unnecessary allocations. I expect at some point Rust will get smart
        // enough that this will no longer be an issue, at which point we should
        // change this to use `match`, as that keeps better with Rust best practices.
        if attribute.name.local_name == "version" {
            version = Some(attribute.value);
        } else if attribute.name.local_name == "base" {
            // TODO: Do we need to validate the URI?
            base_uri = Some(AnyUri(attribute.value));
        } else {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedAttribute {
                    element: "COLLADA",
                    attribute: attribute.name.local_name,
                    expected: COLLADA_ATTRIBS.into(),
                },
            })
        }
    }

    // Verify that all required attributes have been found.
    let version = match version {
        Some(version) => { version }
        None => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingAttribute {
                    element: "COLLADA",
                    attribute: "version",
                },
            })
        }
    };

    if version == "1.4.0" || version == "1.4.1" {
        v1_4::parse_collada(reader, version, base_uri).map(Into::into)
    } else if version == "1.5.0" {
        v1_5::parse_collada(reader, version, base_uri)
    } else {
        Err(Error {
            position: reader.position(),
            kind: ErrorKind::UnsupportedVersion {
                version: version,
            },
        })
    }
}

pub fn required_start_element<R: Read>(
    reader: &mut EventReader<R>,
    parent: &'static str,
    search_name: &'static str,
) -> Result<(OwnedName, Vec<OwnedAttribute>, Namespace)> {
    match reader.next()? {
        StartElement { name, attributes, namespace } => {
            if search_name != name.local_name {
                return Err(Error {
                    position: reader.position(),
                    kind: ErrorKind::UnexpectedElement {
                        element: name.local_name,
                        parent: parent.into(),
                        expected: vec![search_name],
                    },
                })
            }

            return Ok((name, attributes, namespace));
        }

        EndElement { name } => {
            debug_assert_eq!(parent, name.local_name);

            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingElement {
                    parent: parent,
                    expected: search_name,
                },
            })
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent.into(),
                    data: data,
                }
            })
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

// TODO: This should really be `optional_start_element` since it doesn't fail if no element starts,
// but there's already a fn with that name. Once we unify the parsing code we can kill the old one
// and fix the name of this one.
fn start_element<R: Read>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<Option<ElementStart>> {
    match reader.next()? {
        StartElement { name, attributes, namespace: _ } => {
            return Ok(Some(ElementStart { name: name, attributes: attributes }));
        }

        EndElement { name } => {
            debug_assert_eq!(parent, name.local_name);
            return Ok(None);
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent,
                    data: data,
                }
            })
        }

        // TODO: How do we handle processing instructions? I suspect we want to just skip them, but
        // I'm not sure.
        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn required_text_contents<R, T>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<T>
    where
    R: Read,
    T: FromStr,
    ErrorKind: From<<T as FromStr>::Err>
{
    match reader.next()? {
        Characters(data) => {
            let result = T::from_str(&*data)
                .map_err(|error| Error {
                    position: reader.position(),
                    kind: error.into(),
                })?;
            end_element(reader, parent)?;
            return Ok(result);
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        EndElement { .. } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingValue {
                    element: parent,
                },
            });
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn optional_text_contents<R, T>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<Option<T>>
    where
    R: Read,
    T: FromStr,
    ErrorKind: From<<T as FromStr>::Err>
{
    match reader.next()? {
        Characters(data) => {
            let result = T::from_str(&*data)
                .map_err(|error| Error {
                    position: reader.position(),
                    kind: error.into(),
                })?;
            end_element(reader, parent)?;
            return Ok(Some(result));
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        EndElement { .. } => {
            return Ok(None);
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn end_element<R: Read>(reader: &mut EventReader<R>, parent: &'static str) -> Result<()> {
    match reader.next()? {
        EndElement { .. } => {
            return Ok(());
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent.into(),
                    data: data,
                }
            })
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

/// Meaning, of course, "verify that there are no attributes".
pub fn verify_attributes<R: Read>(reader: &EventReader<R>, name: &'static str, attributes: Vec<OwnedAttribute>) -> Result<()> {
    // Make sure the child element has no attributes.
    if attributes.len() != 0 {
        return Err(Error {
            position: reader.position(),
            kind: ErrorKind::UnexpectedAttribute {
                element: name,
                attribute: attributes[0].name.local_name.clone(),
                expected: vec![],
            },
        })
    }

    Ok(())
}

/// Helper struct for pretty-printing lists of strings.
pub struct StringListDisplay<'a>(pub &'a [&'a str]);

impl<'a> Display for StringListDisplay<'a> {
    fn fmt(&self, formatter: &mut Formatter) -> ::std::result::Result<(), fmt::Error> {
        if self.0.len() > 0 {
            write!(formatter, "{}", self.0[0])?;

            for string in &self.0[1..] {
                write!(formatter, ", {}", string)?;
            }
        }

        Ok(())
    }
}
<commit_msg>Support required repeating children<commit_after>use {AnyUri, Result, Error, ErrorKind, v1_4, v1_5};
use self::ChildOccurrences::*;
use std::fmt::{self, Display, Formatter};
use std::io::Read;
use std::str::FromStr;
use xml::attribute::OwnedAttribute;
use xml::common::Position;
use xml::name::OwnedName;
use xml::namespace::Namespace;
use xml::reader::{EventReader, ParserConfig};
use xml::reader::XmlEvent::*;

pub static PARSER_CONFIG: ParserConfig = ParserConfig {
    trim_whitespace: true,
    whitespace_to_characters: true,
    cdata_to_characters: true,
    ignore_comments: true,
    coalesce_characters: true,
};

#[derive(Debug)]
struct ElementStart {
    name: OwnedName,
    attributes: Vec<OwnedAttribute>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChildOccurrences {
    Optional,
    Required,
    Many,
    RequiredMany,
}

pub struct ElementConfiguration<'a, R: 'a + Read> {
    pub name: &'static str,
    pub children: &'a mut [ChildConfiguration<'a, R>],
}

impl<'a, R: 'a + Read> ElementConfiguration<'a, R> {
    pub fn parse(mut self, reader: &mut EventReader<R>) -> Result<()> {
        // Keep track of the text position for the root element so that it can be used for error
        // messages.
        let root_position = reader.position();

        // The index of the next child we are expecting.
        let mut current_child = 0;

        // Whether or not we have encountered the current child at least once. This is only used
        // for `RequiredMany` children to ensure they are found at least once.
        let mut has_encountered_child = false;

        'elements: while let Some(element) = start_element(reader, self.name)? {
            while current_child < self.children.len() {
                let child = &mut self.children[current_child];

                if child.name == element.name.local_name {
                    has_encountered_child = true;

                    // We've found a valid child, hooray! Allow it to run its parsing code.
                    (child.action)(reader, element.attributes)?;

                    // Either advance current_child or don't, depending on if it's allowed to repeat.
                    match child.occurrences {
                        Optional | Required => {
                            // Advance current child.
                            has_encountered_child = false;
                            current_child += 1;
                        }
                        Many | RequiredMany => { /* Don't advance current child. */ }
                    }

                    continue 'elements;
                }

                // The element didn't match the current child. Check to see if the current child
                // is required. If so, we return an error if we never encountered it.
                if child.occurrences == Required || (child.occurrences == RequiredMany && !has_encountered_child) {
                    break;
                }

                // Advance the current child.
                has_encountered_child = false;
                current_child += 1;
            }

            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: self.name,
                    element: element.name.local_name,
                    expected: self.collect_expected_children(),
                },
            });
        }

        // No more child elements are present, and none of the children we encountered were invalid.
        // Verify that there are no remaining required children.
        for child in &self.children[current_child..] {
            if child.occurrences == ChildOccurrences::Required {
                return Err(Error {
                    position: root_position,
                    kind: ErrorKind::MissingElement {
                        parent: self.name,
                        expected: child.name,
                    },
                });
            }
        }

        Ok(())
    }

    fn collect_expected_children(&self) -> Vec<&'static str> {
        let mut names = Vec::with_capacity(self.children.len());
        for child in self.children.iter() {
            names.push(child.name);
        }
        names
    }
}

pub struct ChildConfiguration<'a, R: 'a + Read> {
    pub name: &'static str,
    pub occurrences: ChildOccurrences,
    pub action: &'a mut FnMut(&mut EventReader<R>, Vec<OwnedAttribute>) -> Result<()>,
}

pub fn parse<R: Read>(mut reader: EventReader<R>) -> Result<v1_5::Collada> {
    pub static COLLADA_ATTRIBS: &'static [&'static str] = &["version", "xmlns", "base"];

    // Eat the `StartDocument` event. It has no useful information for our purposes, but it
    // will always be the first event emitted, even if there's no XML declaration at the
    // beginning of the document. This is defined as part of the xml-rs API as of v0.3.5,
    // but it's possible this can will change in the future.
    match reader.next()? {
        StartDocument { .. } => {},
        _ => panic!("First event from EventReader wasn't StartDocument"),
    }

    // The next element will always be the `<COLLADA>` tag. This will specify what version of
    // the COLLADA spec is being used, which is how we'll determine our sub-parser.
    let attributes = match reader.next()? {
        StartElement { name, attributes, namespace: _ } => {
            // If the element isn't the `<COLLADA>` tag then the document is malformed,
            // return an error.
            if name.local_name != "COLLADA" {
                return Err(Error {
                    position: reader.position(),
                    kind: ErrorKind::UnexpectedRootElement {
                        element: name.local_name,
                    }
                })
            }

            attributes
        }

        // I'm *almost* 100% certain that the only event that can follow the `StartDocument`
        // event is a `StartElement` event. As of v0.3.5, xml-rs doesn't support
        // `<!DOCTYPE>` or processing instructions, and it ignores whitespace and comments
        // (according to how we configure the parser), and those are the only things allowed
        // between `StartDocument` and the first `StartElement`. If xml-rs changes its
        // behavior this will need to be updated.
        event @ _ => { panic!("Unexpected event: {:?}", event); }
    };

    // Valide the attributes on the `<COLLADA>` tag.
    // Use boolean flags to track if specific attributes have been encountered.
    let mut version = None;
    let mut base_uri = None;

    for attribute in attributes {
        // NOTE: I'm using `if` blocks instead of `match` here because using `match`
        // won't allow for the name to be moved out of `attribute`. Using `if` saves
        // some unnecessary allocations. I expect at some point Rust will get smart
        // enough that this will no longer be an issue, at which point we should
        // change this to use `match`, as that keeps better with Rust best practices.
        if attribute.name.local_name == "version" {
            version = Some(attribute.value);
        } else if attribute.name.local_name == "base" {
            // TODO: Do we need to validate the URI?
            base_uri = Some(AnyUri(attribute.value));
        } else {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedAttribute {
                    element: "COLLADA",
                    attribute: attribute.name.local_name,
                    expected: COLLADA_ATTRIBS.into(),
                },
            })
        }
    }

    // Verify that all required attributes have been found.
    let version = match version {
        Some(version) => { version }
        None => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingAttribute {
                    element: "COLLADA",
                    attribute: "version",
                },
            })
        }
    };

    if version == "1.4.0" || version == "1.4.1" {
        v1_4::parse_collada(reader, version, base_uri).map(Into::into)
    } else if version == "1.5.0" {
        v1_5::parse_collada(reader, version, base_uri)
    } else {
        Err(Error {
            position: reader.position(),
            kind: ErrorKind::UnsupportedVersion {
                version: version,
            },
        })
    }
}

pub fn required_start_element<R: Read>(
    reader: &mut EventReader<R>,
    parent: &'static str,
    search_name: &'static str,
) -> Result<(OwnedName, Vec<OwnedAttribute>, Namespace)> {
    match reader.next()? {
        StartElement { name, attributes, namespace } => {
            if search_name != name.local_name {
                return Err(Error {
                    position: reader.position(),
                    kind: ErrorKind::UnexpectedElement {
                        element: name.local_name,
                        parent: parent.into(),
                        expected: vec![search_name],
                    },
                })
            }

            return Ok((name, attributes, namespace));
        }

        EndElement { name } => {
            debug_assert_eq!(parent, name.local_name);

            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingElement {
                    parent: parent,
                    expected: search_name,
                },
            })
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent.into(),
                    data: data,
                }
            })
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

// TODO: This should really be `optional_start_element` since it doesn't fail if no element starts,
// but there's already a fn with that name. Once we unify the parsing code we can kill the old one
// and fix the name of this one.
fn start_element<R: Read>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<Option<ElementStart>> {
    match reader.next()? {
        StartElement { name, attributes, namespace: _ } => {
            return Ok(Some(ElementStart { name: name, attributes: attributes }));
        }

        EndElement { name } => {
            debug_assert_eq!(parent, name.local_name);
            return Ok(None);
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent,
                    data: data,
                }
            })
        }

        // TODO: How do we handle processing instructions? I suspect we want to just skip them, but
        // I'm not sure.
        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn required_text_contents<R, T>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<T>
    where
    R: Read,
    T: FromStr,
    ErrorKind: From<<T as FromStr>::Err>
{
    match reader.next()? {
        Characters(data) => {
            let result = T::from_str(&*data)
                .map_err(|error| Error {
                    position: reader.position(),
                    kind: error.into(),
                })?;
            end_element(reader, parent)?;
            return Ok(result);
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        EndElement { .. } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::MissingValue {
                    element: parent,
                },
            });
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn optional_text_contents<R, T>(
    reader: &mut EventReader<R>,
    parent: &'static str,
) -> Result<Option<T>>
    where
    R: Read,
    T: FromStr,
    ErrorKind: From<<T as FromStr>::Err>
{
    match reader.next()? {
        Characters(data) => {
            let result = T::from_str(&*data)
                .map_err(|error| Error {
                    position: reader.position(),
                    kind: error.into(),
                })?;
            end_element(reader, parent)?;
            return Ok(Some(result));
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        EndElement { .. } => {
            return Ok(None);
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

pub fn end_element<R: Read>(reader: &mut EventReader<R>, parent: &'static str) -> Result<()> {
    match reader.next()? {
        EndElement { .. } => {
            return Ok(());
        }

        StartElement { name, attributes: _, namespace: _ } => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedElement {
                    parent: parent,
                    element: name.local_name,
                    expected: vec![],
                },
            })
        }

        Characters(data) => {
            return Err(Error {
                position: reader.position(),
                kind: ErrorKind::UnexpectedCharacterData {
                    element: parent.into(),
                    data: data,
                }
            })
        }

        ProcessingInstruction { .. } => { unimplemented!(); }

        event @ _ => { panic!("Unexpected event: {:?}", event); }
    }
}

/// Meaning, of course, "verify that there are no attributes".
pub fn verify_attributes<R: Read>(reader: &EventReader<R>, name: &'static str, attributes: Vec<OwnedAttribute>) -> Result<()> {
    // Make sure the child element has no attributes.
    if attributes.len() != 0 {
        return Err(Error {
            position: reader.position(),
            kind: ErrorKind::UnexpectedAttribute {
                element: name,
                attribute: attributes[0].name.local_name.clone(),
                expected: vec![],
            },
        })
    }

    Ok(())
}

/// Helper struct for pretty-printing lists of strings.
pub struct StringListDisplay<'a>(pub &'a [&'a str]);

impl<'a> Display for StringListDisplay<'a> {
    fn fmt(&self, formatter: &mut Formatter) -> ::std::result::Result<(), fmt::Error> {
        if self.0.len() > 0 {
            write!(formatter, "{}", self.0[0])?;

            for string in &self.0[1..] {
                write!(formatter, ", {}", string)?;
            }
        }

        Ok(())
    }
}
<|endoftext|>