<commit_before>use crate::{
    aws::Aws,
    health::{Commit, Event, ValidatorEvent},
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    env,
    sync::mpsc,
    thread::{self, JoinHandle},
    time::{Duration, Instant, SystemTime},
};

pub struct AwsLogTail {
    pub event_receiver: mpsc::Receiver<ValidatorEvent>,
    #[allow(dead_code)]
    thread: JoinHandle<()>,
}

struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogTail {
    pub fn spawn_new(aws: Aws) -> failure::Result<Self> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        let thread = builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(AwsLogTail {
            thread,
            event_receiver,
        })
    }

    pub fn recv_all_until_deadline(&self, deadline: Instant) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while Instant::now() < deadline {
            match self.event_receiver.try_recv() {
                Ok(event) => events.push(event),
                Err(..) => thread::sleep(Duration::from_millis(1)),
            }
        }
        events
    }

    pub fn recv_all(&self) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while let Ok(event) = self.event_receiver.try_recv() {
            events.push(event);
        }
        events
    }

    fn log_start_offset_sec() -> u64 {
        let default = 8;
        match env::var("LOG_START_OFFSET") {
            Ok(s) => s.parse().expect("LOG_START_OFFSET env is not a number"),
            Err(..) => default,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let timestamp = SystemTime::now()
            .duration_since(SystemTime::UNIX_EPOCH)
            .expect("now < UNIX_EPOCH");
        let timestamp = timestamp - Duration::from_secs(Self::log_start_offset_sec());
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: "AT_TIMESTAMP".into(),
                stream_name: format!("{}-RecipientStream", aws.workplace()),
                starting_sequence_number: None,
                timestamp: Some(timestamp.as_secs() as f64),
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }
}

impl AwsLogThread {
    fn run(mut self) {
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync()
                .expect("Request to kinesis failed");
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if millis_behind == 0 {
                if let Some(startup_sender) = self.startup_sender.take() {
                    startup_sender.send(()).expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(500))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        let now = SystemTime::now()
            .duration_since(SystemTime::UNIX_EPOCH)
            .expect("Now is behind UNIX_EPOCH");
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let _ignore = self.event_sender.send(ValidatorEvent {
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            });
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        if let Some(cap) = cap {
            Some(cap[1].into())
        } else {
            None
        }
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}
<commit_msg>[cluster_test] Write event log file<commit_after>use crate::{
    aws::Aws,
    health::{Commit, Event, ValidatorEvent},
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    env,
    fs::File,
    io::Write,
    sync::mpsc,
    thread::{self, JoinHandle},
    time::{Duration, Instant, SystemTime},
};

pub struct AwsLogTail {
    pub event_receiver: mpsc::Receiver<ValidatorEvent>,
    #[allow(dead_code)]
    thread: JoinHandle<()>,
}

struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,
    event_log_file: Option<File>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogTail {
    pub fn spawn_new(aws: Aws) -> failure::Result<Self> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let event_log_file = match env::var("EVENT_LOG") {
            Ok(f) => Some(File::create(f).expect("Can't create EVENT_LOG file")),
            Err(..) => None,
        };
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),
            event_log_file,

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        let thread = builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(AwsLogTail {
            thread,
            event_receiver,
        })
    }

    pub fn recv_all_until_deadline(&self, deadline: Instant) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while Instant::now() < deadline {
            match self.event_receiver.try_recv() {
                Ok(event) => events.push(event),
                Err(..) => thread::sleep(Duration::from_millis(1)),
            }
        }
        events
    }

    pub fn recv_all(&self) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while let Ok(event) = self.event_receiver.try_recv() {
            events.push(event);
        }
        events
    }

    fn log_start_offset_sec() -> u64 {
        let default = 8;
        match env::var("LOG_START_OFFSET") {
            Ok(s) => s.parse().expect("LOG_START_OFFSET env is not a number"),
            Err(..) => default,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let timestamp = unix_timestamp_now();
        let timestamp = timestamp - Duration::from_secs(Self::log_start_offset_sec());
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: "AT_TIMESTAMP".into(),
                stream_name: format!("{}-RecipientStream", aws.workplace()),
                starting_sequence_number: None,
                timestamp: Some(timestamp.as_secs() as f64),
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }
}

impl AwsLogThread {
    fn run(mut self) {
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync()
                .expect("Request to kinesis failed");
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            self.write_event_log(format!(
                "Kinesis response, millis behind: {}",
                millis_behind
            ));
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if millis_behind == 0 {
                if let Some(startup_sender) = self.startup_sender.take() {
                    startup_sender.send(()).expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(500))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        self.write_event_log(format!("Kinesis record for {}", validator));
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        let now = unix_timestamp_now();
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn write_event_log(&mut self, s: String) {
        let event_log_file = if let Some(ref mut event_log_file) = self.event_log_file {
            event_log_file
        } else {
            return;
        };
        let now = unix_timestamp_now().as_millis();
        writeln!(event_log_file, "{} {}", now, s).expect("Can't write to EVENT_LOG file");
        event_log_file.flush().expect("Can't flush EVENT_LOG file");
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let ve = ValidatorEvent {
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            };
            self.write_event_log(format!("{:?}", ve));
            let _ignore = self.event_sender.send(ve);
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        if let Some(cap) = cap {
            Some(cap[1].into())
        } else {
            None
        }
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}

fn unix_timestamp_now() -> Duration {
    SystemTime::now()
        .duration_since(SystemTime::UNIX_EPOCH)
        .expect("now < UNIX_EPOCH")
}
<|endoftext|>


<commit_before>extern crate rand;
use std::str;
use std::fmt;
use std::collections::VecDeque;
use rand::Rng;

#[derive(Debug, Eq, PartialEq, PartialOrd, Clone)]
pub struct Card {
    pub suit:  String,
    pub value: String,
}

#[derive(Debug, Eq, PartialEq, PartialOrd, Clone)]
pub struct Deck {
    pub cards: VecDeque<Card>
}

impl Card {
    pub fn new(suit: String, value: String) -> Card {
        Card { suit: suit, value: value.to_string() }
    }

    pub fn new_from_str(suit: &str, value: &str) -> Card {
        Card::new(suit.to_string(), value.to_string())
    }
}

impl Ord for Card {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        self.value.cmp(&other.value)
    }
}

impl fmt::Display for Card {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Suit: {}, Value: {}", self.suit, self.value)
    }
}

impl Deck {
    pub fn from_vec_deque(input: VecDeque<Card>) -> Deck {
        Deck { cards: input }
    }

    pub fn make_standard() -> Deck {
        let mut result = VecDeque::new();
        let suits  = &["♣", "♦", "♥", "♠"];
        let values = &["2", "3", "4", "5", "6", "7", "8", "9", "10",
                       "Jack", "Queen", "King", "Ace"];

        for suit in suits {
            for value in values {
                result.push_back(Card::new_from_str(*suit, value));
            }
        }
        return Deck::from_vec_deque(result);
    }

    pub fn make_fifth_dimension() -> Deck {
        let mut result = VecDeque::new();
        let suits  = &["♣", "♦", "♥", "♠", "★"];
        let values = &["1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
                       "Jack", "Princess", "Queen", "King", "Joker"];

        for suit in suits {
            for value in values {
                result.push_back(Card::new_from_str(*suit, value));
            }
        }
        return Deck::from_vec_deque(result);
    }

    // destructively shuffles the internal vector
    pub fn shuffle(&mut self) {
        // shuffle isn't defined on VecDeques, so it's manually defined here.
        let mut i = self.len();
        while i > 2 {
            i -= 2;
            let j = rand::thread_rng().gen_range(0, i + 1);
            self.cards.swap(i, j)
        }
    }

    // returns a new shuffled deck
    pub fn shuffled(&self) -> Self {
        let mut res = self.clone();
        res.shuffle();
        res
    }

    pub fn len(&self) -> usize {
        self.cards.len()
    }

    pub fn push_front(&mut self, card: Card) {
        self.cards.push_front(card);
    }

    pub fn push_back(&mut self, card: Card) {
        self.cards.push_back(card);
    }

    pub fn pop_front(&mut self) -> Option<Card> {
        self.cards.pop_front()
    }

    pub fn pop_back(&mut self) -> Option<Card> {
        self.cards.pop_back()
    }
}

impl Ord for Deck {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        self.cards.cmp(&other.cards)
    }
}
<commit_msg>add name field and change value from String to i64<commit_after>extern crate rand;
use std::str;
use std::fmt;
use std::collections::VecDeque;
use rand::Rng;

#[derive(Debug, Eq, PartialEq, PartialOrd, Clone)]
pub struct Card {
    pub name:  String,
    pub suit:  String,
    pub value: i64,
}

#[derive(Debug, Eq, PartialEq, PartialOrd, Clone)]
pub struct Deck {
    pub cards: VecDeque<Card>
}

impl Card {
    pub fn new(suit: String, name: String, value: i64) -> Card {
        Card { suit: suit, name: name, value: value }
    }

    pub fn new_from_str(suit: &str, name: &str, value: i64) -> Card {
        Card::new(suit.to_string(), name.to_string(), value)
    }
}

impl Ord for Card {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        self.value.cmp(&other.value)
    }
}

impl fmt::Display for Card {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Suit: {}, Name: {}", self.suit, self.name)
    }
}

impl Deck {
    pub fn from_vec_deque(input: VecDeque<Card>) -> Deck {
        Deck { cards: input }
    }

    pub fn make_standard() -> Deck {
        let mut result = VecDeque::new();
        let suits  = &["♣", "♦", "♥", "♠"];
        let values = &[
            (2, "2"),     (3, "3"),      (4, "4"),
            (5, "5"),     (6, "6"),      (7, "7"),
            (8, "8"),     (9, "9"),      (10, "10"),
            (11, "Jack"), (12, "Queen"), (13, "King"), (14, "Ace")
        ];

        for suit in suits {
            for &(value, name) in values {
                result.push_back(Card::new_from_str(*suit, name, value));
            }
        }
        return Deck::from_vec_deque(result);
    }

    pub fn make_fifth_dimension() -> Deck {
        let mut result = VecDeque::new();
        let suits  = &["♣", "♦", "♥", "♠", "★"];
        let values = &[
            (1, "1"),      (2, "2"),     (3, "3"),     (4, "4"),
            (5, "5"),      (6, "6"),     (7, "7"),     (8, "8"),
            (9, "9"),      (10, "10"),   (11, "Jack"), (12, "Princess"),
            (13, "Queen"), (14, "King"), (15, "Ace"),  (16, "Joker"),
        ];

        for suit in suits {
            for &(value, name) in values {
                result.push_back(Card::new_from_str(*suit, name, value));
            }
        }
        return Deck::from_vec_deque(result);
    }

    // destructively shuffles the internal vector
    pub fn shuffle(&mut self) {
        // shuffle isn't defined on VecDeques, so it's manually defined here.
        let mut i = self.len();
        while i > 2 {
            i -= 2;
            let j = rand::thread_rng().gen_range(0, i + 1);
            self.cards.swap(i, j)
        }
    }

    // returns a new shuffled deck
    pub fn shuffled(&self) -> Self {
        let mut res = self.clone();
        res.shuffle();
        res
    }

    pub fn len(&self) -> usize {
        self.cards.len()
    }

    pub fn push_front(&mut self, card: Card) {
        self.cards.push_front(card);
    }

    pub fn push_back(&mut self, card: Card) {
        self.cards.push_back(card);
    }

    pub fn pop_front(&mut self) -> Option<Card> {
        self.cards.pop_front()
    }

    pub fn pop_back(&mut self) -> Option<Card> {
        self.cards.pop_back()
    }
}

impl Ord for Deck {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        self.cards.cmp(&other.cards)
    }
}
<|endoftext|>


<commit_before>// Hound -- A WAV encoding library in Rust
// Copyright (C) 2015 Ruud van Asseldonk
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 3 of the License only.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//! Hound, a WAV encoding library.
//!
//! TODO: Add some examples here.

#![warn(missing_docs)]
#![allow(dead_code)] // TODO: Remove for v0.1
#![feature(io)]

use std::io;
use std::io::{Seek, Write};
use std::marker;

trait WriteExt: io::Write {
    fn write_le_u16(&mut self, x: u16) -> io::Result<()>;
    fn write_le_u32(&mut self, x: u32) -> io::Result<()>;
}

impl<W> WriteExt for W where W: io::Write {

    fn write_le_u16(&mut self, x: u16) -> io::Result<()> {
        let mut buf = [0u8; 2];
        buf[0] = (x & 0xff) as u8;
        buf[1] = (x >> 8) as u8;
        self.write_all(&buf)
    }

    fn write_le_u32(&mut self, x: u32) -> io::Result<()> {
        let mut buf = [0u8; 4];
        buf[0] = ((x >> 00) & 0xff) as u8;
        buf[1] = ((x >> 08) & 0xff) as u8;
        buf[2] = ((x >> 16) & 0xff) as u8;
        buf[3] = ((x >> 24) & 0xff) as u8;
        self.write_all(&buf)
    }
}

trait Sample {
    fn write<W: io::Write>(self, writer: &mut W) -> io::Result<()>;
}

impl Sample for u16 {
    fn write<W: io::Write>(self, writer: &mut W) -> io::Result<()> {
        writer.write_le_u16(self)
    }
}

struct WavWriter<W, S> {
    writer: W,
    wrote_header: bool,
    phantom_sample: marker::PhantomData<fn(S)>
}

impl<W, S> WavWriter<W, S>
where W: io::Write + io::Seek,
      S: Sample {

    pub fn with_bps(writer: W,
                    n_channels: u16,
                    sample_rate: u32,
                    bits_per_sample: u32)
                    -> WavWriter<W, S> {
        unimplemented!();
    }

    pub fn new(writer: W,
               n_channels: u16,
               sample_rate: u32)
               -> WavWriter<W, S> {
        use std::mem::size_of;
        WavWriter::with_bps(writer,
                            n_channels,
                            sample_rate,
                            size_of::<S> as u32 * 8)
    }

    /// Writes the RIFF WAVE header
    fn write_header(&mut self) -> io::Result<()> {
        let mut header = [0u8; 44];

        // Write the header in-memory first.
        {
            let mut buffer: io::Cursor<&mut [u8]> = io::Cursor::new(&mut header);
            try!(buffer.write_all("RIFF".as_bytes()));

            // Skip 4 bytes that will be filled with the file size afterwards.
            buffer.seek(io::SeekFrom::Current(4));

            try!(buffer.write_all("WAVE".as_bytes()));
            try!(buffer.write_all("fmt\0".as_bytes()));
            try!(buffer.write_le_u32(16)); // Size of the WAVE header
            try!(buffer.write_le_u16(1));  // PCM encoded audio
            try!(buffer.write_le_u16(2)); // TODO: num channels
            try!(buffer.write_le_u32(44100)); // TODO: sample rate
            try!(buffer.write_le_u32(0)); // TODO: sr * bps * n_chan / 8
            try!(buffer.write_le_u16(16)); // TODO: block align
            try!(buffer.write_le_u32(16)); // TODO: bits per sample
        }

        // Then write the entire header at once.
        try!(self.writer.write_all(&header));

        Ok(())
    }

    pub fn write_sample(sample: S) -> io::Result<()> {
        // TODO
        Ok(())
    }
}
<commit_msg>add constructor that writes to a file, group options in struct<commit_after>// Hound -- A WAV encoding library in Rust
// Copyright (C) 2015 Ruud van Asseldonk
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 3 of the License only.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//! Hound, a WAV encoding library.
//!
//! TODO: Add some examples here.

#![warn(missing_docs)]
#![allow(dead_code)] // TODO: Remove for v0.1
#![feature(convert, io)]

use std::fs;
use std::io;
use std::io::{Seek, Write};
use std::marker;
use std::path;

trait WriteExt: io::Write {
    fn write_le_u16(&mut self, x: u16) -> io::Result<()>;
    fn write_le_u32(&mut self, x: u32) -> io::Result<()>;
}

impl<W> WriteExt for W where W: io::Write {

    fn write_le_u16(&mut self, x: u16) -> io::Result<()> {
        let mut buf = [0u8; 2];
        buf[0] = (x & 0xff) as u8;
        buf[1] = (x >> 8) as u8;
        self.write_all(&buf)
    }

    fn write_le_u32(&mut self, x: u32) -> io::Result<()> {
        let mut buf = [0u8; 4];
        buf[0] = ((x >> 00) & 0xff) as u8;
        buf[1] = ((x >> 08) & 0xff) as u8;
        buf[2] = ((x >> 16) & 0xff) as u8;
        buf[3] = ((x >> 24) & 0xff) as u8;
        self.write_all(&buf)
    }
}

trait Sample {
    fn write<W: io::Write>(self, writer: &mut W) -> io::Result<()>;
}

impl Sample for u16 {
    fn write<W: io::Write>(self, writer: &mut W) -> io::Result<()> {
        writer.write_le_u16(self)
    }
}

pub struct WavSpec {
    n_channels: u16,
    sample_rate: u16,
    bits_per_sample: u16
}

struct WavWriter<W, S> {
    writer: W,
    wrote_header: bool,
    phantom_sample: marker::PhantomData<fn(S)>
}

impl<W, S> WavWriter<W, S>
where W: io::Write + io::Seek,
      S: Sample {

    /// Creates a writer that writes the WAVE format to the underlying writer.
    pub fn new(writer: W, spec: WavSpec) -> WavWriter<W, S> {
        unimplemented!();
    }

    /// Creates a writer that writes the WAVE format to a file.
    ///
    /// The file will be overwritten if it exists.
    pub fn create<P: AsRef<path::Path>>(filename: P, spec: WavSpec)
           -> io::Result<WavWriter<fs::File, S>> {
        let file = try!(fs::File::create(filename));
        Ok(WavWriter::new(file, spec))
    }

    /// Writes the RIFF WAVE header
    fn write_header(&mut self) -> io::Result<()> {
        let mut header = [0u8; 44];

        // Write the header in-memory first.
        {
            let mut buffer: io::Cursor<&mut [u8]> = io::Cursor::new(&mut header);
            try!(buffer.write_all("RIFF".as_bytes()));

            // Skip 4 bytes that will be filled with the file size afterwards.
            buffer.seek(io::SeekFrom::Current(4));

            try!(buffer.write_all("WAVE".as_bytes()));
            try!(buffer.write_all("fmt\0".as_bytes()));
            try!(buffer.write_le_u32(16)); // Size of the WAVE header
            try!(buffer.write_le_u16(1));  // PCM encoded audio
            try!(buffer.write_le_u16(2)); // TODO: num channels
            try!(buffer.write_le_u32(44100)); // TODO: sample rate
            try!(buffer.write_le_u32(0)); // TODO: sr * bps * n_chan / 8
            try!(buffer.write_le_u16(16)); // TODO: block align
            try!(buffer.write_le_u32(16)); // TODO: bits per sample
        }

        // Then write the entire header at once.
        try!(self.writer.write_all(&header));

        Ok(())
    }

    pub fn write_sample(sample: S) -> io::Result<()> {
        // TODO
        Ok(())
    }
}
<|endoftext|>


<commit_before>//! MySQL Proxy Server

#[macro_use]
extern crate log;
extern crate env_logger;
#[macro_use]
extern crate futures;
#[macro_use]
extern crate tokio_core;
extern crate futures_cpupool;
extern crate byteorder;

use std::cell::RefCell;
use std::rc::Rc;
use std::env;
use std::io::{self, Read, Write, Error, ErrorKind};
use std::net::{SocketAddr, Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6};
use std::net::Shutdown;
use std::str;
use std::time::Duration;

use futures::{Future, Poll, Async};
use futures::stream::Stream;
use futures_cpupool::CpuPool;
use tokio_core::net::{TcpStream, TcpListener};
use tokio_core::io::{copy, Io};
use tokio_core::reactor::Core;
use byteorder::*;

#[derive(Copy,Clone)]
pub enum PacketType {
    ComSleep = 0x00,
    ComQuit = 0x01,
    ComInitDb = 0x02,
    ComQuery = 0x03,
    ComFieldList = 0x04,
    ComCreateDb = 0x05,
    ComDropDb = 0x06,
    ComRefresh = 0x07,
    ComShutdown = 0x08,
    ComStatistics = 0x09,
    ComProcessInfo = 0x0a,
    ComConnect = 0x0b,
    ComProcessKill= 0x0c,
    ComDebug = 0x0d,
    ComPing = 0x0e,
    ComTime = 0x0f,
    ComDelayedInsert = 0x10,
    ComChangeUser = 0x11,
    ComBinlogDump = 0x12,
    ComTableDump = 0x13,
    ComConnectOut = 0x14,
    ComRegisterSlave = 0x15,
    ComStmtPrepare = 0x16,
    ComStmtExecute = 0x17,
    ComStmtSendLongData = 0x18,
    ComStmtClose = 0x19,
    ComStmtReset = 0x1a,
    ComDaemon= 0x1d,
    ComBinlogDumpGtid = 0x1e,
    ComResetConnection = 0x1f,
}

pub struct Packet {
    pub bytes: Vec<u8>
}

impl Packet {

    pub fn error_packet(code: u16, state: [u8; 5], msg: String) -> Self {

        // start building payload
        let mut payload: Vec<u8> = Vec::with_capacity(9 + msg.len());
        payload.push(0xff);  // packet type
        payload.write_u16::<LittleEndian>(code).unwrap(); // error code
        payload.extend_from_slice("#".as_bytes()); // sql_state_marker
        payload.extend_from_slice(&state); // SQL STATE
        payload.extend_from_slice(msg.as_bytes());

        // create header with length and sequence id
        let mut header: Vec<u8> = Vec::with_capacity(4 + 9 + msg.len());
        header.write_u32::<LittleEndian>(payload.len() as u32).unwrap();
        header.pop(); // we need 3 byte length, so discard last byte
        header.push(1); // sequence_id

        // combine the vectors
        header.extend_from_slice(&payload);

        // now move the vector into the packet
        Packet { bytes: header }
    }

    pub fn sequence_id(&self) -> u8 {
        self.bytes[3]
    }

    //TODO: should return Result<PacketType, ?>
    pub fn packet_type(&self) -> PacketType {
        match self.bytes[4] {
            0x00 => PacketType::ComSleep,
            0x01 => PacketType::ComQuit,
            0x02 => PacketType::ComInitDb,
            0x03 => PacketType::ComQuery,
            0x04 => PacketType::ComFieldList,
            0x05 => PacketType::ComCreateDb,
            0x06 => PacketType::ComDropDb,
            0x07 => PacketType::ComRefresh,
            0x08 => PacketType::ComShutdown,
            0x09 => PacketType::ComStatistics,
            0x0a => PacketType::ComProcessInfo,
            0x0b => PacketType::ComConnect,
            0x0c => PacketType::ComProcessKill,
            0x0d => PacketType::ComDebug,
            0x0e => PacketType::ComPing,
            0x0f => PacketType::ComTime,
            0x10 => PacketType::ComDelayedInsert,
            0x11 => PacketType::ComChangeUser,
            0x12 => PacketType::ComBinlogDump,
            0x13 => PacketType::ComTableDump,
            0x14 => PacketType::ComConnectOut,
            0x15 => PacketType::ComRegisterSlave,
            0x16 => PacketType::ComStmtPrepare,
            0x17 => PacketType::ComStmtExecute,
            0x18 => PacketType::ComStmtSendLongData,
            0x19 => PacketType::ComStmtClose,
            0x1a => PacketType::ComStmtReset,
            0x1d => PacketType::ComDaemon,
            0x1e => PacketType::ComBinlogDumpGtid,
            0x1f => PacketType::ComResetConnection,
            _ => panic!("Unsupported packet type")
        }
    }

}

pub enum Action {
    Forward,                // forward the packet unmodified
    Mutate(Packet),         // mutate the packet
    Respond(Vec<Packet>)    // handle directly and optionally return some packets
}

pub trait PacketHandler {
    fn handle_request(&self, p: &Packet) -> Action;
    fn handle_response(&self, p: &Packet) -> Action;
}

fn parse_packet_length(header: &[u8]) -> usize {
    (((header[2] as u32) << 16) |
    ((header[1] as u32) << 8) |
    header[0] as u32) as usize
}

pub struct Proxy<H> where H: PacketHandler {
    handler: H
}

impl<H> Proxy<H> where H: PacketHandler {

    pub fn run() {

        env_logger::init().unwrap();
        let addr = env::args().nth(1).unwrap_or("127.0.0.1:8080".to_string());
        let addr = addr.parse::<SocketAddr>().unwrap();

        // Create the event loop that will drive this server
        let mut l = Core::new().unwrap();
        let handle = l.handle();

        // Create a TCP listener which will listen for incoming connections
        let socket = TcpListener::bind(&addr, &l.handle()).unwrap();

        // Once we've got the TCP listener, inform that we have it
        println!("Listening on: {}", addr);

        // Pull out the stream of incoming connections and then for each new
        // one spin up a new task copying data.
        //
        // We use the `io::copy` future to copy all data from the
        // reading half onto the writing half.
        let done = socket.incoming().for_each(move |(socket, addr)| {
            let pair = futures::lazy(|| futures::finished(socket.split()));
            let amt = pair.and_then(|(reader, writer)| copy(reader, writer));

            // Once all that is done we print out how much we wrote, and then
            // critically we *spawn* this future which allows it to run
            // concurrently with other connections.
            let msg = amt.map(move |amt| {
                println!("wrote {} bytes to {}", amt, addr)
            }).map_err(|e| {
                panic!("error: {}", e);
            });
            handle.spawn(msg);

            Ok(())
        });
        l.run(done).unwrap();
    }

//    pub fn serve<H: PacketHandler + 'static>(self, conn: TcpStream,
//                                             handler: H,
//                                        ) -> Box<Future<Item = (u64, u64), Error = Error>> {
//
//        // connect to MySQL
//        let addr = Ipv4Addr::new(127, 0, 0, 1);
//        let port = 3306;
//        let addr = SocketAddr::V4(SocketAddrV4::new(addr, port));
//
//        let handle = self.handle.clone();
//        let connected = handle.tcp_connect(&addr);
//
//        let pair = connected.and_then(move |server| Ok((conn,server)));
//
//        Box::new(pair.and_then(move |(c1, c2)| {
//            let c1 = Rc::new(c1);
//            let c2 = Rc::new(c2);
//            Pipe::new(c1, c2, handler)
//        }))
//    }
}

#[derive(Debug)]
enum Direction {
    Client,
    Server
}

struct ConnReader {
    direction: Direction,
    stream: Rc<TcpStream>,
    read_buf: Vec<u8>,
    read_pos: usize,
    read_amt: u64,
}

struct ConnWriter {
    direction: Direction,
    stream: Rc<TcpStream>,
    write_buf: Vec<u8>,
    write_pos: usize,
    write_amt: u64,
}

impl ConnReader {

    fn new(stream: Rc<TcpStream>, direction: Direction) -> Self {
        ConnReader {
            stream: stream,
            direction: direction,
            read_buf: vec![0u8; 4096],
            read_pos: 0,
            read_amt: 0,
        }
    }

    /// Read from the socket until the status is NotReady
    fn read(&mut self) -> Poll<(), io::Error> {
        loop {
            try_ready!(self.stream.poll_read());
            //TODO: ensure capacity first
            let n = try_nb!((&*self.stream).read(&mut self.read_buf[self.read_pos..]));
            if n == 0 {
                return Err(Error::new(ErrorKind::Other, "connection closed"));
            }
            self.read_amt += n as u64;
            self.read_pos += n;
        }
    }

    fn next(&mut self) -> Option<Packet> {
        // do we have a header
        if self.read_pos > 3 {
            let l = parse_packet_length(&self.read_buf);
            // do we have the whole packet?
            let s = 4 + l;
            if self.read_pos >= s {
                let mut temp : Vec<u8> = Vec::with_capacity(s);
                temp.extend_from_slice(&self.read_buf[0..s]);
                let p = Packet { bytes: temp };

                // shift data down
                let mut j = 0;
                for i in s .. self.read_pos {
                    self.read_buf[j] = self.read_buf[i];
                    j += 1;
                }
                self.read_pos -= s;

                Some(p)
            } else {
                None
            }
        } else {
            None
        }
    }
}

impl ConnWriter {

    fn new(stream: Rc<TcpStream>, direction: Direction) -> Self {
        ConnWriter{
            stream: stream,
            direction: direction,
            write_buf: vec![0u8; 4096],
            write_pos: 0,
            write_amt: 0
        }
    }

    /// Write a packet to the write buffer
    fn push(&mut self, p: &Packet) {
        for i in 0 .. p.bytes.len() {
            self.write_buf[self.write_pos + i] = p.bytes[i];
        }
        self.write_pos += p.bytes.len();
    }

    /// Writes the contents of the write buffer to the socket
    fn write(&mut self) -> Poll<(), io::Error> {
        while self.write_pos > 0 {
            try_ready!(self.stream.poll_write());
            let s = try!((&*self.stream).write(&self.write_buf[0..self.write_pos]));

            let mut j = 0;
            for i in s .. self.write_pos {
                self.write_buf[j] = self.write_buf[i];
                j += 1;
            }
            self.write_pos -= s;

        }
        return Ok(Async::Ready(()));
    }
}

struct Pipe<H: PacketHandler + 'static> {
    client_reader: ConnReader,
    client_writer: ConnWriter,
    server_reader: ConnReader,
    server_writer: ConnWriter,
    handler: H,
}

impl<H> Pipe<H> where H: PacketHandler + 'static {
    fn new(client: Rc<TcpStream>,
           server: Rc<TcpStream>,
           handler: H
           ) -> Pipe<H> {

        Pipe {
            client_reader: ConnReader::new(client.clone(), Direction::Client),
            client_writer: ConnWriter::new(client, Direction::Client),
            server_reader: ConnReader::new(server.clone(), Direction::Server),
            server_writer: ConnWriter::new(server, Direction::Server),
            handler: handler,
        }
    }
}

impl<H> Future for Pipe<H> where H: PacketHandler + 'static {
    type Item = (u64, u64);
    type Error = Error;

    fn poll(&mut self) -> Poll<(u64, u64), io::Error> {
        loop {
            let client_read = self.client_reader.read();

            // if the client connection has closed, close the server connection too
            match &client_read {
                &Err(ref e) => { self.server_writer.stream.shutdown(Shutdown::Write).unwrap(); },
                _ => {}
            }

            // process buffered requests
            while let Some(request) = self.client_reader.next() {
                match self.handler.handle_request(&request) {
                    Action::Forward => self.server_writer.push(&request),
                    Action::Mutate(ref p2) => self.server_writer.push(p2),
                    Action::Respond(ref v) => {
                        for p in v {
                            self.client_writer.push(&p);
                        }
                    }
                };
            }

            // try reading from server
            let server_read = self.server_reader.read();

            // if the server connection has closed, close the client connection too
            match &server_read {
                &Err(ref e) => { self.client_writer.stream.shutdown(Shutdown::Write).unwrap(); },
                _ => {}
            }

            // process buffered responses
            while let Some(response) = self.server_reader.next() {
                match self.handler.handle_response(&response) {
                    Action::Forward => self.client_writer.push(&response),
                    Action::Mutate(ref p2) => self.client_writer.push(p2),
                    Action::Respond(ref v) => {
                        for p in v {
                            self.server_writer.push(&p);
                        }
                    }
                };
            }

            // perform all of the writes at the end, since the request handlers may have
            // queued packets in either, or both directions

            // try writing to client
            let client_write = self.client_writer.write();

            // try writing to server
            let server_write = self.server_writer.write();

            try_ready!(client_read);
            try_ready!(client_write);
            try_ready!(server_read);
            try_ready!(server_write);
        }

    }

}

#[allow(dead_code)]
pub fn print_packet_chars(buf: &[u8]) {
    print!("[");
    for i in 0..buf.len() {
        print!("{} ", buf[i] as char);
    }
    println!("]");
}

#[allow(dead_code)]
pub fn print_packet_bytes(buf: &[u8]) {
    print!("[");
    for i in 0..buf.len() {
        if i%8==0 { println!(""); }
        print!("{:#04x} ",buf[i]);
    }
    println!("]");
}

<commit_msg>updating to use tokio 0.1.0<commit_after>//! MySQL Proxy Server

#[macro_use]
extern crate log;
extern crate env_logger;
#[macro_use]
extern crate futures;
#[macro_use]
extern crate tokio_core;
extern crate futures_cpupool;
extern crate byteorder;

use std::cell::RefCell;
use std::rc::Rc;
use std::env;
use std::io::{self, Read, Write, Error, ErrorKind};
use std::net::{SocketAddr, Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6};
use std::net::Shutdown;
use std::str;
use std::time::Duration;

use futures::{Future, Poll, Async};
use futures::stream::Stream;
use futures_cpupool::CpuPool;
use tokio_core::net::{TcpStream, TcpListener};
use tokio_core::io::{copy, Io};
use tokio_core::reactor::Core;
use byteorder::*;

#[derive(Copy,Clone)]
pub enum PacketType {
    ComSleep = 0x00,
    ComQuit = 0x01,
    ComInitDb = 0x02,
    ComQuery = 0x03,
    ComFieldList = 0x04,
    ComCreateDb = 0x05,
    ComDropDb = 0x06,
    ComRefresh = 0x07,
    ComShutdown = 0x08,
    ComStatistics = 0x09,
    ComProcessInfo = 0x0a,
    ComConnect = 0x0b,
    ComProcessKill= 0x0c,
    ComDebug = 0x0d,
    ComPing = 0x0e,
    ComTime = 0x0f,
    ComDelayedInsert = 0x10,
    ComChangeUser = 0x11,
    ComBinlogDump = 0x12,
    ComTableDump = 0x13,
    ComConnectOut = 0x14,
    ComRegisterSlave = 0x15,
    ComStmtPrepare = 0x16,
    ComStmtExecute = 0x17,
    ComStmtSendLongData = 0x18,
    ComStmtClose = 0x19,
    ComStmtReset = 0x1a,
    ComDaemon= 0x1d,
    ComBinlogDumpGtid = 0x1e,
    ComResetConnection = 0x1f,
}

pub struct Packet {
    pub bytes: Vec<u8>
}

impl Packet {

    pub fn error_packet(code: u16, state: [u8; 5], msg: String) -> Self {

        // start building payload
        let mut payload: Vec<u8> = Vec::with_capacity(9 + msg.len());
        payload.push(0xff);  // packet type
        payload.write_u16::<LittleEndian>(code).unwrap(); // error code
        payload.extend_from_slice("#".as_bytes()); // sql_state_marker
        payload.extend_from_slice(&state); // SQL STATE
        payload.extend_from_slice(msg.as_bytes());

        // create header with length and sequence id
        let mut header: Vec<u8> = Vec::with_capacity(4 + 9 + msg.len());
        header.write_u32::<LittleEndian>(payload.len() as u32).unwrap();
        header.pop(); // we need 3 byte length, so discard last byte
        header.push(1); // sequence_id

        // combine the vectors
        header.extend_from_slice(&payload);

        // now move the vector into the packet
        Packet { bytes: header }
    }

    pub fn sequence_id(&self) -> u8 {
        self.bytes[3]
    }

    //TODO: should return Result<PacketType, ?>
    pub fn packet_type(&self) -> PacketType {
        match self.bytes[4] {
            0x00 => PacketType::ComSleep,
            0x01 => PacketType::ComQuit,
            0x02 => PacketType::ComInitDb,
            0x03 => PacketType::ComQuery,
            0x04 => PacketType::ComFieldList,
            0x05 => PacketType::ComCreateDb,
            0x06 => PacketType::ComDropDb,
            0x07 => PacketType::ComRefresh,
            0x08 => PacketType::ComShutdown,
            0x09 => PacketType::ComStatistics,
            0x0a => PacketType::ComProcessInfo,
            0x0b => PacketType::ComConnect,
            0x0c => PacketType::ComProcessKill,
            0x0d => PacketType::ComDebug,
            0x0e => PacketType::ComPing,
            0x0f => PacketType::ComTime,
            0x10 => PacketType::ComDelayedInsert,
            0x11 => PacketType::ComChangeUser,
            0x12 => PacketType::ComBinlogDump,
            0x13 => PacketType::ComTableDump,
            0x14 => PacketType::ComConnectOut,
            0x15 => PacketType::ComRegisterSlave,
            0x16 => PacketType::ComStmtPrepare,
            0x17 => PacketType::ComStmtExecute,
            0x18 => PacketType::ComStmtSendLongData,
            0x19 => PacketType::ComStmtClose,
            0x1a => PacketType::ComStmtReset,
            0x1d => PacketType::ComDaemon,
            0x1e => PacketType::ComBinlogDumpGtid,
            0x1f => PacketType::ComResetConnection,
            _ => panic!("Unsupported packet type")
        }
    }

}

pub enum Action {
    Forward,                // forward the packet unmodified
    Mutate(Packet),         // mutate the packet
    Respond(Vec<Packet>)    // handle directly and optionally return some packets
}

pub trait PacketHandler {
    fn handle_request(&self, p: &Packet) -> Action;
    fn handle_response(&self, p: &Packet) -> Action;
}

fn parse_packet_length(header: &[u8]) -> usize {
    (((header[2] as u32) << 16) |
    ((header[1] as u32) << 8) |
    header[0] as u32) as usize
}

pub struct Proxy<H> where H: PacketHandler {
    handler: H
}

impl<H> Proxy<H> where H: PacketHandler {

    pub fn run() {

        env_logger::init().unwrap();
        let addr = env::args().nth(1).unwrap_or("127.0.0.1:8080".to_string());
        let addr = addr.parse::<SocketAddr>().unwrap();

        // Create the event loop that will drive this server
        let mut l = Core::new().unwrap();
        let handle = l.handle();

        // Create a TCP listener which will listen for incoming connections
        let socket = TcpListener::bind(&addr, &l.handle()).unwrap();

        // Once we've got the TCP listener, inform that we have it
        println!("Listening on: {}", addr);

        // Pull out the stream of incoming connections and then for each new
        // one spin up a new task copying data.
        //
        // We use the `io::copy` future to copy all data from the
        // reading half onto the writing half.
        let done = socket.incoming().for_each(move |(socket, addr)| {
            let pair = futures::lazy(|| futures::finished(socket.split()));
            let amt = pair.and_then(|(reader, writer)| copy(reader, writer));

            // Once all that is done we print out how much we wrote, and then
            // critically we *spawn* this future which allows it to run
            // concurrently with other connections.
            let msg = amt.map(move |amt| {
                println!("wrote {} bytes to {}", amt, addr)
            }).map_err(|e| {
                panic!("error: {}", e);
            });
            handle.spawn(msg);

            Ok(())
        });
        l.run(done).unwrap();
    }

//    pub fn serve<H: PacketHandler + 'static>(self, conn: TcpStream,
//                                             handler: H,
//                                        ) -> Box<Future<Item = (u64, u64), Error = Error>> {
//
//        // connect to MySQL
//        let addr = Ipv4Addr::new(127, 0, 0, 1);
//        let port = 3306;
//        let addr = SocketAddr::V4(SocketAddrV4::new(addr, port));
//
//        let handle = self.handle.clone();
//        let connected = handle.tcp_connect(&addr);
//
//        let pair = connected.and_then(move |server| Ok((conn,server)));
//
//        Box::new(pair.and_then(move |(c1, c2)| {
//            let c1 = Rc::new(c1);
//            let c2 = Rc::new(c2);
//            Pipe::new(c1, c2, handler)
//        }))
//    }
}

#[derive(Debug)]
enum Direction {
    Client,
    Server
}

struct ConnReader {
    direction: Direction,
    stream: Rc<TcpStream>,
    read_buf: Vec<u8>,
    read_pos: usize,
    read_amt: u64,
}

struct ConnWriter {
    direction: Direction,
    stream: Rc<TcpStream>,
    write_buf: Vec<u8>,
    write_pos: usize,
    write_amt: u64,
}

impl ConnReader {

    fn new(stream: Rc<TcpStream>, direction: Direction) -> Self {
        ConnReader {
            stream: stream,
            direction: direction,
            read_buf: vec![0u8; 4096],
            read_pos: 0,
            read_amt: 0,
        }
    }

    /// Read from the socket until the status is NotReady
    fn read(&mut self) -> Poll<(), io::Error> {
        loop {
            match self.stream.poll_read() {
                Async::Ready(_) => {
                    //TODO: ensure capacity first
                    let n = try_nb!((&*self.stream).read(&mut self.read_buf[self.read_pos..]));
                    if n == 0 {
                        return Err(Error::new(ErrorKind::Other, "connection closed"));
                    }
                    self.read_amt += n as u64;
                    self.read_pos += n;
                },
                _ => return Ok(Async::NotReady),
            }
        }
    }

    fn next(&mut self) -> Option<Packet> {
        // do we have a header
        if self.read_pos > 3 {
            let l = parse_packet_length(&self.read_buf);
            // do we have the whole packet?
            let s = 4 + l;
            if self.read_pos >= s {
                let mut temp : Vec<u8> = Vec::with_capacity(s);
                temp.extend_from_slice(&self.read_buf[0..s]);
                let p = Packet { bytes: temp };

                // shift data down
                let mut j = 0;
                for i in s .. self.read_pos {
                    self.read_buf[j] = self.read_buf[i];
                    j += 1;
                }
                self.read_pos -= s;

                Some(p)
            } else {
                None
            }
        } else {
            None
        }
    }
}

impl ConnWriter {

    fn new(stream: Rc<TcpStream>, direction: Direction) -> Self {
        ConnWriter{
            stream: stream,
            direction: direction,
            write_buf: vec![0u8; 4096],
            write_pos: 0,
            write_amt: 0
        }
    }

    /// Write a packet to the write buffer
    fn push(&mut self, p: &Packet) {
        for i in 0 .. p.bytes.len() {
            self.write_buf[self.write_pos + i] = p.bytes[i];
        }
        self.write_pos += p.bytes.len();
    }

    /// Writes the contents of the write buffer to the socket
    fn write(&mut self) -> Poll<(), io::Error> {
        while self.write_pos > 0 {
            match self.stream.poll_write() {
                Async::Ready(_) => {
                    let s = try!((&*self.stream).write(&self.write_buf[0..self.write_pos]));

                    let mut j = 0;
                    for i in s..self.write_pos {
                        self.write_buf[j] = self.write_buf[i];
                        j += 1;
                    }
                    self.write_pos -= s;
                },
                _ => return Ok(Async::NotReady)
            }
        }
        return Ok(Async::Ready(()));
    }
}

struct Pipe<H: PacketHandler + 'static> {
    client_reader: ConnReader,
    client_writer: ConnWriter,
    server_reader: ConnReader,
    server_writer: ConnWriter,
    handler: H,
}

impl<H> Pipe<H> where H: PacketHandler + 'static {
    fn new(client: Rc<TcpStream>,
           server: Rc<TcpStream>,
           handler: H
           ) -> Pipe<H> {

        Pipe {
            client_reader: ConnReader::new(client.clone(), Direction::Client),
            client_writer: ConnWriter::new(client, Direction::Client),
            server_reader: ConnReader::new(server.clone(), Direction::Server),
            server_writer: ConnWriter::new(server, Direction::Server),
            handler: handler,
        }
    }
}

impl<H> Future for Pipe<H> where H: PacketHandler + 'static {
    type Item = (u64, u64);
    type Error = Error;

    fn poll(&mut self) -> Poll<(u64, u64), io::Error> {
        loop {
            let client_read = self.client_reader.read();

            // if the client connection has closed, close the server connection too
            match &client_read {
                &Err(ref e) => { self.server_writer.stream.shutdown(Shutdown::Write).unwrap(); },
                _ => {}
            }

            // process buffered requests
            while let Some(request) = self.client_reader.next() {
                match self.handler.handle_request(&request) {
                    Action::Forward => self.server_writer.push(&request),
                    Action::Mutate(ref p2) => self.server_writer.push(p2),
                    Action::Respond(ref v) => {
                        for p in v {
                            self.client_writer.push(&p);
                        }
                    }
                };
            }

            // try reading from server
            let server_read = self.server_reader.read();

            // if the server connection has closed, close the client connection too
            match &server_read {
                &Err(ref e) => { self.client_writer.stream.shutdown(Shutdown::Write).unwrap(); },
                _ => {}
            }

            // process buffered responses
            while let Some(response) = self.server_reader.next() {
                match self.handler.handle_response(&response) {
                    Action::Forward => self.client_writer.push(&response),
                    Action::Mutate(ref p2) => self.client_writer.push(p2),
                    Action::Respond(ref v) => {
                        for p in v {
                            self.server_writer.push(&p);
                        }
                    }
                };
            }

            // perform all of the writes at the end, since the request handlers may have
            // queued packets in either, or both directions

            // try writing to client
            let client_write = self.client_writer.write();

            // try writing to server
            let server_write = self.server_writer.write();

            try_ready!(client_read);
            try_ready!(client_write);
            try_ready!(server_read);
            try_ready!(server_write);
        }

    }

}

#[allow(dead_code)]
pub fn print_packet_chars(buf: &[u8]) {
    print!("[");
    for i in 0..buf.len() {
        print!("{} ", buf[i] as char);
    }
    println!("]");
}

#[allow(dead_code)]
pub fn print_packet_bytes(buf: &[u8]) {
    print!("[");
    for i in 0..buf.len() {
        if i%8==0 { println!(""); }
        print!("{:#04x} ",buf[i]);
    }
    println!("]");
}

<|endoftext|>


<commit_before>#![feature(collections, core, io, path, os)]

use std::old_io::Command;
use std::old_io::process::InheritFd;
use std::default::Default;

/// Extra configuration to pass to gcc.
pub struct Config {
    /// Directories where gcc will look for header files.
    pub include_directories: Vec<Path>,
    /// Additional definitions (`-DKEY` or `-DKEY=VALUE`).
    pub definitions: Vec<(String, Option<String>)>,
    /// Additional object files to link into the final archive
    pub objects: Vec<Path>,
    /// Additional flags and parameter to pass to the compiler
    pub flags: Vec<String>,
}

impl Default for Config {
    fn default() -> Config {
        Config {
            include_directories: Vec::new(),
            definitions: Vec::new(),
            objects: Vec::new(),
            flags: Vec::new(),
        }
    }
}

fn getenv(v: &str) -> Option<String> {
    use std::os::getenv;
    let r = getenv(v);
    println!("{} = {:?}", v, r);
    r
}

/// Compile a library from the given set of input C files.
///
/// This will simply compile all files into object files and then assemble them
/// into the output. This will read the standard environment variables to detect
/// cross compilations and such.
///
/// # Example
///
/// ```no_run
/// use std::default::Default;
/// gcc::compile_library("libfoo.a", &Default::default(), &[
///     "foo.c",
///     "bar.c",
/// ]);
/// ```
pub fn compile_library(output: &str, config: &Config, files: &[&str]) {
    assert!(output.starts_with("lib"));
    assert!(output.ends_with(".a"));

    let target = getenv("TARGET").unwrap();
    let opt_level = getenv("OPT_LEVEL").unwrap();

    let mut cmd = Command::new(gcc(target.as_slice()));
    cmd.arg(format!("-O{}", opt_level));
    cmd.arg("-c");
    cmd.arg("-ffunction-sections").arg("-fdata-sections");
    cmd.args(cflags().as_slice());

    if target.as_slice().contains("-ios") {
        cmd.args(ios_flags(target.as_slice()).as_slice());
    } else {
        if target.contains("windows") {
            cmd.arg("-mwin32");
        }

        if target.as_slice().contains("i686") {
            cmd.arg("-m32");
        } else if target.as_slice().contains("x86_64") {
            cmd.arg("-m64");
        }

        if !target.as_slice().contains("i686") {
            cmd.arg("-fPIC");
        }
    }

    for directory in config.include_directories.iter() {
        cmd.arg("-I").arg(directory);
    }

    for flag in config.flags.iter() {
        cmd.arg(flag);
    }

    for &(ref key, ref value) in config.definitions.iter() {
        if let &Some(ref value) = value {
            cmd.arg(format!("-D{}={}", key, value));
        } else {
            cmd.arg(format!("-D{}", key));
        }
    }

    let src = Path::new(getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(getenv("OUT_DIR").unwrap());
    let mut objects = Vec::new();
    for file in files.iter() {
        let obj = dst.join(*file).with_extension("o");
        std::old_io::fs::mkdir_recursive(&obj.dir_path(), std::old_io::USER_RWX).unwrap();
        run(cmd.clone().arg(src.join(*file)).arg("-o").arg(&obj));
        objects.push(obj);
    }

    run(Command::new(ar(target.as_slice())).arg("crus")
                                           .arg(dst.join(output))
                                           .args(objects.as_slice())
                                           .args(config.objects.as_slice()));
    println!("cargo:rustc-flags=-L native={} -l static={}",
             dst.display(), output.slice(3, output.len() - 2));
}

fn run(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    let status = match cmd.stdout(InheritFd(1)).stderr(InheritFd(2)).status() {
        Ok(status) => status,
        Err(e) => panic!("failed to spawn process: {}", e),
    };
    if !status.success() {
        panic!("nonzero exit status: {}", status);
    }
}

fn get_var(var_base: &str) -> Result<String, String> {
    let target = getenv("TARGET")
        .expect("Environment variable 'TARGET' is unset");
    let host = match getenv("HOST") {
            None => { return Err("Environment variable 'HOST' is unset".to_string()); }
            Some(x) => x
        };
    let kind = if host == target { "HOST" } else { "TARGET" };
    let target_u = target.split('-')
        .collect::<Vec<&str>>()
        .connect("_");
    let res = getenv(format!("{}_{}", var_base, target).as_slice())
        .or_else(|| getenv(format!("{}_{}", var_base, target_u).as_slice()))
        .or_else(|| getenv(format!("{}_{}", kind, var_base).as_slice()))
        .or_else(|| getenv(var_base));

    match res {
        Some(res) => Ok(res),
        None => Err("Could not get environment variable".to_string()),
    }
}

fn gcc(target: &str) -> String {
    let is_android = target.find_str("android").is_some();

    get_var("CC").unwrap_or(if cfg!(windows) {
        "gcc".to_string()
    } else if is_android {
        format!("{}-gcc", target)
    } else {
        "cc".to_string()
    })
}

fn ar(target: &str) -> String {
    let is_android = target.find_str("android").is_some();

    get_var("AR").unwrap_or(if is_android {
        format!("{}-ar", target)
    } else {
        "ar".to_string()
    })
}

fn cflags() -> Vec<String> {
    get_var("CFLAGS").unwrap_or(String::new())
       .as_slice().words().map(|s| s.to_string())
       .collect()
}

fn ios_flags(target: &str) -> Vec<String> {
    enum ArchSpec {
        Device(&'static str),
        Simulator(&'static str),
    }

    let mut res = Vec::new();

    let arch = target.split('-').nth(0).expect("expected target in format `arch-vendor-os`");
    let arch = match arch {
        "arm" | "armv7" | "thumbv7" => ArchSpec::Device("armv7"),
        "armv7s" | "thumbv7s" => ArchSpec::Device("armv7s"),
        "arm64" | "aarch64" => ArchSpec::Device("aarch64"),
        "i386" | "i686" => ArchSpec::Simulator("-m32"),
        "x86_64" => ArchSpec::Simulator("-m64"),
        _ => unreachable!("Unknown arch for iOS target")
    };

    let sdk = match arch {
        ArchSpec::Device(arch) => {
            res.push("-arch".to_string());
            res.push(arch.to_string());
            "iphoneos"
        },
        ArchSpec::Simulator(arch) => {
            res.push(arch.to_string());
            "iphonesimulator"
        }
    };

    println!("Detecting iOS SDK path for {}", sdk);
    let sdk_path = Command::new("xcrun")
        .arg("--show-sdk-path")
        .arg("--sdk")
        .arg(sdk)
        .stderr(InheritFd(2))
        .output()
        .unwrap()
        .output;

    let sdk_path = String::from_utf8(sdk_path).unwrap();

    res.push("-isysroot".to_string());
    res.push(sdk_path.trim().to_string());

    res
}
<commit_msg>Move to var_string<commit_after>#![feature(core, io, path, env)]

use std::default::Default;
use std::env;
use std::old_io::Command;
use std::old_io::process::InheritFd;

/// Extra configuration to pass to gcc.
pub struct Config {
    /// Directories where gcc will look for header files.
    pub include_directories: Vec<Path>,
    /// Additional definitions (`-DKEY` or `-DKEY=VALUE`).
    pub definitions: Vec<(String, Option<String>)>,
    /// Additional object files to link into the final archive
    pub objects: Vec<Path>,
    /// Additional flags and parameter to pass to the compiler
    pub flags: Vec<String>,
}

impl Default for Config {
    fn default() -> Config {
        Config {
            include_directories: Vec::new(),
            definitions: Vec::new(),
            objects: Vec::new(),
            flags: Vec::new(),
        }
    }
}

fn getenv(v: &str) -> Option<String> {
    let r = env::var_string(v).ok();
    println!("{} = {:?}", v, r);
    r
}

/// Compile a library from the given set of input C files.
///
/// This will simply compile all files into object files and then assemble them
/// into the output. This will read the standard environment variables to detect
/// cross compilations and such.
///
/// # Example
///
/// ```no_run
/// use std::default::Default;
/// gcc::compile_library("libfoo.a", &Default::default(), &[
///     "foo.c",
///     "bar.c",
/// ]);
/// ```
pub fn compile_library(output: &str, config: &Config, files: &[&str]) {
    assert!(output.starts_with("lib"));
    assert!(output.ends_with(".a"));

    let target = getenv("TARGET").unwrap();
    let opt_level = getenv("OPT_LEVEL").unwrap();

    let mut cmd = Command::new(gcc(target.as_slice()));
    cmd.arg(format!("-O{}", opt_level));
    cmd.arg("-c");
    cmd.arg("-ffunction-sections").arg("-fdata-sections");
    cmd.args(cflags().as_slice());

    if target.as_slice().contains("-ios") {
        cmd.args(ios_flags(target.as_slice()).as_slice());
    } else {
        if target.contains("windows") {
            cmd.arg("-mwin32");
        }

        if target.as_slice().contains("i686") {
            cmd.arg("-m32");
        } else if target.as_slice().contains("x86_64") {
            cmd.arg("-m64");
        }

        if !target.as_slice().contains("i686") {
            cmd.arg("-fPIC");
        }
    }

    for directory in config.include_directories.iter() {
        cmd.arg("-I").arg(directory);
    }

    for flag in config.flags.iter() {
        cmd.arg(flag);
    }

    for &(ref key, ref value) in config.definitions.iter() {
        if let &Some(ref value) = value {
            cmd.arg(format!("-D{}={}", key, value));
        } else {
            cmd.arg(format!("-D{}", key));
        }
    }

    let src = Path::new(getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(getenv("OUT_DIR").unwrap());
    let mut objects = Vec::new();
    for file in files.iter() {
        let obj = dst.join(*file).with_extension("o");
        std::old_io::fs::mkdir_recursive(&obj.dir_path(), std::old_io::USER_RWX).unwrap();
        run(cmd.clone().arg(src.join(*file)).arg("-o").arg(&obj));
        objects.push(obj);
    }

    run(Command::new(ar(target.as_slice())).arg("crus")
                                           .arg(dst.join(output))
                                           .args(objects.as_slice())
                                           .args(config.objects.as_slice()));
    println!("cargo:rustc-flags=-L native={} -l static={}",
             dst.display(), output.slice(3, output.len() - 2));
}

fn run(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    let status = match cmd.stdout(InheritFd(1)).stderr(InheritFd(2)).status() {
        Ok(status) => status,
        Err(e) => panic!("failed to spawn process: {}", e),
    };
    if !status.success() {
        panic!("nonzero exit status: {}", status);
    }
}

fn get_var(var_base: &str) -> Result<String, String> {
    let target = getenv("TARGET")
        .expect("Environment variable 'TARGET' is unset");
    let host = match getenv("HOST") {
            None => { return Err("Environment variable 'HOST' is unset".to_string()); }
            Some(x) => x
        };
    let kind = if host == target { "HOST" } else { "TARGET" };
    let target_u = target.split('-')
        .collect::<Vec<&str>>()
        .connect("_");
    let res = getenv(format!("{}_{}", var_base, target).as_slice())
        .or_else(|| getenv(format!("{}_{}", var_base, target_u).as_slice()))
        .or_else(|| getenv(format!("{}_{}", kind, var_base).as_slice()))
        .or_else(|| getenv(var_base));

    match res {
        Some(res) => Ok(res),
        None => Err("Could not get environment variable".to_string()),
    }
}

fn gcc(target: &str) -> String {
    let is_android = target.find_str("android").is_some();

    get_var("CC").unwrap_or(if cfg!(windows) {
        "gcc".to_string()
    } else if is_android {
        format!("{}-gcc", target)
    } else {
        "cc".to_string()
    })
}

fn ar(target: &str) -> String {
    let is_android = target.find_str("android").is_some();

    get_var("AR").unwrap_or(if is_android {
        format!("{}-ar", target)
    } else {
        "ar".to_string()
    })
}

fn cflags() -> Vec<String> {
    get_var("CFLAGS").unwrap_or(String::new())
       .as_slice().words().map(|s| s.to_string())
       .collect()
}

fn ios_flags(target: &str) -> Vec<String> {
    enum ArchSpec {
        Device(&'static str),
        Simulator(&'static str),
    }

    let mut res = Vec::new();

    let arch = target.split('-').nth(0).expect("expected target in format `arch-vendor-os`");
    let arch = match arch {
        "arm" | "armv7" | "thumbv7" => ArchSpec::Device("armv7"),
        "armv7s" | "thumbv7s" => ArchSpec::Device("armv7s"),
        "arm64" | "aarch64" => ArchSpec::Device("aarch64"),
        "i386" | "i686" => ArchSpec::Simulator("-m32"),
        "x86_64" => ArchSpec::Simulator("-m64"),
        _ => unreachable!("Unknown arch for iOS target")
    };

    let sdk = match arch {
        ArchSpec::Device(arch) => {
            res.push("-arch".to_string());
            res.push(arch.to_string());
            "iphoneos"
        },
        ArchSpec::Simulator(arch) => {
            res.push(arch.to_string());
            "iphonesimulator"
        }
    };

    println!("Detecting iOS SDK path for {}", sdk);
    let sdk_path = Command::new("xcrun")
        .arg("--show-sdk-path")
        .arg("--sdk")
        .arg(sdk)
        .stderr(InheritFd(2))
        .output()
        .unwrap()
        .output;

    let sdk_path = String::from_utf8(sdk_path).unwrap();

    res.push("-isysroot".to_string());
    res.push(sdk_path.trim().to_string());

    res
}
<|endoftext|>


<commit_before>// Copyright 2015 Corey Farwell
// Copyright 2015 Contributors of github.com/huonw/crates.io-graph
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Library for retrieving and interacting with the [crates.io index].
//!
//! [crates.io index]: https://github.com/rust-lang/crates.io-index
//!
//! ## Examples
//!
//! ```rust
//! extern crate crates_index;
//!
//! let index = crates_index::Index::new("_index".into());
//! if !index.exists() {
//!    index.retrieve().expect("Could not fetch crates.io index");
//! }
//! for crate_ in index.crates() {
//!    let latest_version = crate_.latest_version();
//!    println!("crate name: {}", latest_version.name());
//!    println!("crate version: {}", latest_version.version());
//! }
//! ```

use std::ascii::AsciiExt;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};

extern crate git2;
extern crate glob;
extern crate rustc_serialize;

static INDEX_GIT_URL: &'static str = "https://github.com/rust-lang/crates.io-index";

/// A single version of a crate published to the index
#[derive(RustcDecodable, Clone)]
pub struct Version {
    name: String,
    vers: String,
    deps: Vec<Dependency>,
    cksum: String,
    features: HashMap<String, Vec<String>>,
    yanked: bool,
}

impl Version {
    /// Name of the crate
    pub fn name(&self) -> &str {
        &self.name
    }

    /// Name of this version
    pub fn version(&self) -> &str {
        &self.vers
    }

    /// Dependencies for this version
    pub fn dependencies(&self) -> &[Dependency] {
        &self.deps
    }

    /// Checksum of the package for this version
    pub fn checksum(&self) -> &str {
        &self.cksum
    }

    pub fn features(&self) -> &HashMap<String, Vec<String>> {
        &self.features
    }

    /// Whether this version was [yanked](http://doc.crates.io/crates-io.html#cargo-yank) from the
    /// index
    pub fn is_yanked(&self) -> bool {
        self.yanked
    }
}

/// A single dependency of a specific crate version
#[derive(RustcDecodable, Clone)]
pub struct Dependency {
    name: String,
    req: String,
    features: Vec<String>,
    optional: bool,
    default_features: bool,
    target: Option<String>,
    kind: Option<String>,
}

impl Dependency {
    pub fn name(&self) -> &str {
        &self.name
    }

    pub fn requirement(&self) -> &str {
        &self.req
    }

    pub fn features(&self) -> &[String] {
        &self.features
    }

    pub fn is_optional(&self) -> bool {
        self.optional
    }

    pub fn has_default_features(&self) -> bool {
        self.default_features
    }

    pub fn target(&self) -> Option<&str> {
        match self.target {
            Some(ref s) => Some(s),
            None => None,
        }
    }

    pub fn kind(&self) -> Option<&str> {
        match self.kind {
            Some(ref s) => Some(s),
            None => None,
        }
    }
}

/// Constructed from `Index::crates`
pub struct Crates(CrateIndexPaths);

impl Iterator for Crates {
    type Item = Crate;
    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|p| Crate::new(&p))
    }
}

/// Constructed from `Index::crate_index_paths`
pub struct CrateIndexPaths(std::iter::Chain<glob::Paths, glob::Paths>);

impl CrateIndexPaths {
    fn new<P: AsRef<Path>>(path: P) -> CrateIndexPaths {
        let mut match_options = glob::MatchOptions::new();
        match_options.require_literal_leading_dot = true;
        let path = path.as_ref();

        let glob_pattern = format!("{}/*/*/*", path.to_str().unwrap());
        let index_paths1 = glob::glob_with(&glob_pattern, &match_options).unwrap();

        let glob_pattern = format!("{}/[12]/*", path.to_str().unwrap());
        let index_paths2 = glob::glob_with(&glob_pattern, &match_options).unwrap();

        CrateIndexPaths(index_paths1.chain(index_paths2))
    }
}

impl Iterator for CrateIndexPaths {
    type Item = PathBuf;
    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|glob_result| glob_result.unwrap())
    }
}

/// Wrapper around managing the crates.io-index git repository
pub struct Index {
    path: PathBuf,
}

impl Index {
    /// Construct a new Index supplying a path where the index lives or should live
    pub fn new(path: PathBuf) -> Index {
        Index { path: path }
    }

    /// Determines if *anything* exists at the path specified from the constructor
    pub fn exists(&self) -> bool {
        fs::metadata(&self.path).is_ok()
    }

    /// Downloads the index to the path specified from the constructor
    pub fn retrieve(&self) -> Result<(), git2::Error> {
        let _ = try!(git2::Repository::clone(INDEX_GIT_URL, &self.path));
        Ok(())
    }

    /// Retrieve a single crate by name (case insensitive) from the index
    pub fn crate_(&self, crate_name: &str) -> Option<Crate> {
        self.crate_index_paths()
            .find(|path| {
                      path.file_name()
                          .and_then(OsStr::to_str)
                          .map(|file_name| file_name.eq_ignore_ascii_case(crate_name))
                          .unwrap_or(false)
                  })
            .map(|p| Crate::new(&p))
    }

    /// Retrieve an iterator over all the crates in the index
    pub fn crates(&self) -> Crates {
        Crates(self.crate_index_paths())
    }

    /// Returns all the crate index file paths in the index
    pub fn crate_index_paths(&self) -> CrateIndexPaths {
        CrateIndexPaths::new(self.path.clone()) // TODO: remove this clone
    }
}

/// A single crate that contains many published versions
pub struct Crate {
    versions: Vec<Version>,
}

impl Crate {
    pub fn new<P: AsRef<Path>>(index_path: P) -> Crate {
        let index_path = index_path.as_ref();
        let mut versions = vec![];
        let file = fs::File::open(&index_path).unwrap();
        for line in BufReader::new(file).lines() {
            let version: Version = rustc_serialize::json::decode(&line.unwrap()).unwrap();
            versions.push(version);
        }
        Crate { versions: versions }
    }

    /// Published versions of this crate sorted chronologically by date published
    pub fn versions(&self) -> &[Version] {
        &self.versions
    }

    pub fn earliest_version(&self) -> &Version {
        &self.versions[0]
    }

    pub fn latest_version(&self) -> &Version {
        &self.versions[self.versions.len() - 1]
    }
}

#[test]
fn test_dependencies() {
    let index = Index::new("_test".into());
    if !index.exists() {
        index.retrieve().unwrap();
    }
    let crate_ = index.crates().nth(0).unwrap();
    let version = crate_.latest_version();
    let _ = version.deps;
}
<commit_msg>Harden `exists` checking.<commit_after>// Copyright 2015 Corey Farwell
// Copyright 2015 Contributors of github.com/huonw/crates.io-graph
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Library for retrieving and interacting with the [crates.io index].
//!
//! [crates.io index]: https://github.com/rust-lang/crates.io-index
//!
//! ## Examples
//!
//! ```rust
//! extern crate crates_index;
//!
//! let index = crates_index::Index::new("_index".into());
//! if !index.exists() {
//!    index.retrieve().expect("Could not fetch crates.io index");
//! }
//! for crate_ in index.crates() {
//!    let latest_version = crate_.latest_version();
//!    println!("crate name: {}", latest_version.name());
//!    println!("crate version: {}", latest_version.version());
//! }
//! ```

use std::ascii::AsciiExt;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};

extern crate git2;
extern crate glob;
extern crate rustc_serialize;

static INDEX_GIT_URL: &'static str = "https://github.com/rust-lang/crates.io-index";

/// A single version of a crate published to the index
#[derive(RustcDecodable, Clone)]
pub struct Version {
    name: String,
    vers: String,
    deps: Vec<Dependency>,
    cksum: String,
    features: HashMap<String, Vec<String>>,
    yanked: bool,
}

impl Version {
    /// Name of the crate
    pub fn name(&self) -> &str {
        &self.name
    }

    /// Name of this version
    pub fn version(&self) -> &str {
        &self.vers
    }

    /// Dependencies for this version
    pub fn dependencies(&self) -> &[Dependency] {
        &self.deps
    }

    /// Checksum of the package for this version
    pub fn checksum(&self) -> &str {
        &self.cksum
    }

    pub fn features(&self) -> &HashMap<String, Vec<String>> {
        &self.features
    }

    /// Whether this version was [yanked](http://doc.crates.io/crates-io.html#cargo-yank) from the
    /// index
    pub fn is_yanked(&self) -> bool {
        self.yanked
    }
}

/// A single dependency of a specific crate version
#[derive(RustcDecodable, Clone)]
pub struct Dependency {
    name: String,
    req: String,
    features: Vec<String>,
    optional: bool,
    default_features: bool,
    target: Option<String>,
    kind: Option<String>,
}

impl Dependency {
    pub fn name(&self) -> &str {
        &self.name
    }

    pub fn requirement(&self) -> &str {
        &self.req
    }

    pub fn features(&self) -> &[String] {
        &self.features
    }

    pub fn is_optional(&self) -> bool {
        self.optional
    }

    pub fn has_default_features(&self) -> bool {
        self.default_features
    }

    pub fn target(&self) -> Option<&str> {
        match self.target {
            Some(ref s) => Some(s),
            None => None,
        }
    }

    pub fn kind(&self) -> Option<&str> {
        match self.kind {
            Some(ref s) => Some(s),
            None => None,
        }
    }
}

/// Constructed from `Index::crates`
pub struct Crates(CrateIndexPaths);

impl Iterator for Crates {
    type Item = Crate;
    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|p| Crate::new(&p))
    }
}

/// Constructed from `Index::crate_index_paths`
pub struct CrateIndexPaths(std::iter::Chain<glob::Paths, glob::Paths>);

impl CrateIndexPaths {
    fn new<P: AsRef<Path>>(path: P) -> CrateIndexPaths {
        let mut match_options = glob::MatchOptions::new();
        match_options.require_literal_leading_dot = true;
        let path = path.as_ref();

        let glob_pattern = format!("{}/*/*/*", path.to_str().unwrap());
        let index_paths1 = glob::glob_with(&glob_pattern, &match_options).unwrap();

        let glob_pattern = format!("{}/[12]/*", path.to_str().unwrap());
        let index_paths2 = glob::glob_with(&glob_pattern, &match_options).unwrap();

        CrateIndexPaths(index_paths1.chain(index_paths2))
    }
}

impl Iterator for CrateIndexPaths {
    type Item = PathBuf;
    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|glob_result| glob_result.unwrap())
    }
}

/// Wrapper around managing the crates.io-index git repository
pub struct Index {
    path: PathBuf,
}

impl Index {
    /// Construct a new Index supplying a path where the index lives or should live
    pub fn new(path: PathBuf) -> Index {
        Index { path: path }
    }

    /// Determines if a crates.io repository exists at `self.path`
    pub fn exists(&self) -> bool {
        git2::Repository::discover(&self.path)
            .map(|repository| {
                repository
                    .find_remote("origin")
                    .map(|remote| remote.url() == Some(INDEX_GIT_URL))
                    .unwrap_or(false)
            })
            .unwrap_or(false)
    }

    /// Downloads the index to the path specified from the constructor
    pub fn retrieve(&self) -> Result<(), git2::Error> {
        let _ = try!(git2::Repository::clone(INDEX_GIT_URL, &self.path));
        Ok(())
    }

    /// Retrieve a single crate by name (case insensitive) from the index
    pub fn crate_(&self, crate_name: &str) -> Option<Crate> {
        self.crate_index_paths()
            .find(|path| {
                      path.file_name()
                          .and_then(OsStr::to_str)
                          .map(|file_name| file_name.eq_ignore_ascii_case(crate_name))
                          .unwrap_or(false)
                  })
            .map(|p| Crate::new(&p))
    }

    /// Retrieve an iterator over all the crates in the index
    pub fn crates(&self) -> Crates {
        Crates(self.crate_index_paths())
    }

    /// Returns all the crate index file paths in the index
    pub fn crate_index_paths(&self) -> CrateIndexPaths {
        CrateIndexPaths::new(self.path.clone()) // TODO: remove this clone
    }
}

/// A single crate that contains many published versions
pub struct Crate {
    versions: Vec<Version>,
}

impl Crate {
    pub fn new<P: AsRef<Path>>(index_path: P) -> Crate {
        let index_path = index_path.as_ref();
        let mut versions = vec![];
        let file = fs::File::open(&index_path).unwrap();
        for line in BufReader::new(file).lines() {
            let version: Version = rustc_serialize::json::decode(&line.unwrap()).unwrap();
            versions.push(version);
        }
        Crate { versions: versions }
    }

    /// Published versions of this crate sorted chronologically by date published
    pub fn versions(&self) -> &[Version] {
        &self.versions
    }

    pub fn earliest_version(&self) -> &Version {
        &self.versions[0]
    }

    pub fn latest_version(&self) -> &Version {
        &self.versions[self.versions.len() - 1]
    }
}

#[cfg(test)]
mod test {
    use std::fs;
    use super::Index;

    static TEST_INDEX_DIR: &'static str = "_test";

    #[test]
    fn test_dependencies() {
        let _ = fs::remove_dir_all(TEST_INDEX_DIR);

        let index = Index::new(TEST_INDEX_DIR.into());
        if !index.exists() {
            index.retrieve().unwrap();
        }
        let crate_ = index.crates().nth(0).unwrap();
        let version = crate_.latest_version();
        let _ = version.deps;

        let _ = fs::remove_dir_all(TEST_INDEX_DIR);
    }

    #[test]
    fn test_exists() {
        let _ = fs::remove_dir_all(TEST_INDEX_DIR);

        let index = Index::new(TEST_INDEX_DIR.into());
        assert!(!index.exists());
        index.retrieve().unwrap();
        assert!(index.exists());

        let _ = fs::remove_dir_all(TEST_INDEX_DIR);
    }
}
<|endoftext|>