<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # DFA compiler
//!
//! The DFA compiler converts NDFAs into DFAs, using a DFA builder.
//!

use std::marker::PhantomData;

use super::dfa_builder::*;
use super::state_machine::*;

///
/// Builds a deterministic finite automaton from a NDFA
///
pub struct DfaCompiler<InputSymbol: PartialOrd, OutputSymbol, DfaType, Ndfa: StateMachine<InputSymbol, OutputSymbol>, Builder: DfaBuilder<InputSymbol, OutputSymbol, DfaType>> {
    ndfa: Ndfa,
    builder: Builder,

    // Phantom data to poke Rust's type system (it's too dumb to see that InputSymbol is used in both Ndfa and Builder there via the type constraint)
    #[allow(dead_code)]
    phantom: (PhantomData<InputSymbol>, PhantomData<OutputSymbol>, PhantomData<DfaType>)
}

impl<InputSymbol: PartialOrd, OutputSymbol, DfaType, Ndfa: StateMachine<InputSymbol, OutputSymbol>, Builder: DfaBuilder<InputSymbol, OutputSymbol, DfaType>>
    DfaCompiler<InputSymbol, OutputSymbol, DfaType, Ndfa, Builder> {
    pub fn new(ndfa: Ndfa, builder: Builder) -> Self {
        DfaCompiler { ndfa: ndfa, builder: builder, phantom: (PhantomData, PhantomData, PhantomData) }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::regular_pattern::*;
    use super::super::state_machine::*;
    use super::super::symbol_range_dfa::*;

    #[test]
    fn create_compiler() {
        let ndfa = "abc".into_pattern().to_ndfa("success");
        let builder = SymbolRangeDfaBuilder::new();

        let compiler = DfaCompiler::new(ndfa, builder);
    }
}<commit_msg>Update comments, add straight through build function<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # DFA compiler
//!
//! The DFA compiler converts NDFAs into DFAs, using a DFA builder.
//!

use std::marker::PhantomData;

use super::dfa_builder::*;
use super::state_machine::*;

///
/// Builds a deterministic finite automaton from a NDFA
///
pub struct DfaCompiler<InputSymbol: PartialOrd, OutputSymbol, DfaType, Ndfa: StateMachine<InputSymbol, OutputSymbol>, Builder: DfaBuilder<InputSymbol, OutputSymbol, DfaType>> {
    /// State machine that is to be compiled
    ndfa: Ndfa,

    /// Builder where the state machine should be generated
    builder: Builder,

    // Phantom data to poke Rust's type system (it's too dumb to see that InputSymbol is used in both Ndfa and Builder there via the type constraint)
    #[allow(dead_code)]
    phantom: (PhantomData<InputSymbol>, PhantomData<OutputSymbol>, PhantomData<DfaType>)
}

impl<InputSymbol: PartialOrd, OutputSymbol, DfaType, Ndfa: StateMachine<InputSymbol, OutputSymbol>, Builder: DfaBuilder<InputSymbol, OutputSymbol, DfaType>>
    DfaCompiler<InputSymbol, OutputSymbol, DfaType, Ndfa, Builder> {
    ///
    /// Builds a DFA using an NDFA and a builder
    ///
    pub fn build(ndfa: Ndfa, builder: Builder) -> DfaType {
        let compiler = DfaCompiler::new(ndfa, builder);
        compiler.compile()
    }

    ///
    /// Creates a new DFA compiler using a particular builder and NDFA
    ///
    pub fn new(ndfa: Ndfa, builder: Builder) -> Self {
        DfaCompiler { ndfa: ndfa, builder: builder, phantom: (PhantomData, PhantomData, PhantomData) }
    }

    ///
    /// Compiles the NDFA into a DFA
    ///
    pub fn compile(self) -> DfaType {
        unimplemented!()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::regular_pattern::*;
    use super::super::state_machine::*;
    use super::super::symbol_range_dfa::*;

    #[test]
    fn can_create_compiler() {
        let ndfa     = "abc".into_pattern().to_ndfa("success");
        let builder  = SymbolRangeDfaBuilder::new();

        DfaCompiler::new(ndfa, builder);
    }

    #[test]
    fn can_build_dfa() {
        let ndfa     = "abc".into_pattern().to_ndfa("success");
        let builder  = SymbolRangeDfaBuilder::new();

        DfaCompiler::build(ndfa, builder);
    }
}
<|endoftext|>


<commit_before>use ansi_term::Colour;
use fern;
use log::{LogLevel, LogLevelFilter};
use std::io;
use chrono;

use xivdb::error::*;

fn colored_level(level: LogLevel) -> String {
  let color = match level {
    LogLevel::Trace => Colour::Fixed(8),
    LogLevel::Info => Colour::Blue,
    LogLevel::Warn => Colour::Yellow,
    LogLevel::Error => Colour::Red,
    _ => return level.to_string()
  };
  color.paint(level.to_string()).to_string()
}

fn colored_target(target: &str) -> String {
  let parts: Vec<&str> = target.split("::").collect();
  if parts.len() == 1 {
    return target.to_string();
  }
  let base = &parts[..parts.len() - 1];
  let target = &parts[parts.len() - 1];

  let separator = Colour::Fixed(8).paint("::").to_string();
  let mut colored = Vec::new();
  for part in base {
    colored.push(*part);
    colored.push(&separator);
  }
  colored.push(*target);
  colored.join("")
}

pub fn init_logger() -> Result<()> {
  fern::Dispatch::new()
    .format(|out, message, record| {
      out.finish(format_args!("[{}] [{}] {} – {}",
        chrono::Local::now().format("%H:%M:%S"),
        colored_level(record.level()),
        colored_target(record.target()),
        message))
    })
    .level(LogLevelFilter::Info)
    .chain(io::stdout())
    .apply()
    .chain_err(|| "could not set up logger")
}
<commit_msg>fix: use environment variable to turn on debug mode<commit_after>use ansi_term::Colour;
use fern;
use log::{LogLevel, LogLevelFilter};
use std::io;
use chrono;
use std::env::var;

use xivdb::error::*;

fn colored_level(level: LogLevel) -> String {
  let color = match level {
    LogLevel::Trace => Colour::Fixed(8),
    LogLevel::Info => Colour::Blue,
    LogLevel::Warn => Colour::Yellow,
    LogLevel::Error => Colour::Red,
    _ => return level.to_string()
  };
  color.paint(level.to_string()).to_string()
}

fn colored_target(target: &str) -> String {
  let parts: Vec<&str> = target.split("::").collect();
  if parts.len() == 1 {
    return target.to_string();
  }
  let base = &parts[..parts.len() - 1];
  let target = &parts[parts.len() - 1];

  let separator = Colour::Fixed(8).paint("::").to_string();
  let mut colored = Vec::new();
  for part in base {
    colored.push(*part);
    colored.push(&separator);
  }
  colored.push(*target);
  colored.join("")
}

pub fn init_logger() -> Result<()> {
  fern::Dispatch::new()
    .format(|out, message, record| {
      out.finish(format_args!("[{}] [{}] {} – {}",
        chrono::Local::now().format("%H:%M:%S"),
        colored_level(record.level()),
        colored_target(record.target()),
        message))
    })
    .level(if var("LB_DEBUG").is_ok() { LogLevelFilter::Debug } else { LogLevelFilter::Info })
    .chain(io::stdout())
    .apply()
    .chain_err(|| "could not set up logger")
}
<|endoftext|>


<commit_before>//! Traits, implementations, and macros related to logging timely events.

use std::rc::Rc;
use std::cell::RefCell;
use std::sync::Arc;

use ::progress::timestamp::RootTimestamp;
use ::progress::nested::product::Product;
use ::progress::frontier::MutableAntichain;

use dataflow::operators::capture::{Event, EventPusher};

use timely_communication::logging::{BufferingLogger, LoggerBatch, CommsEvent, CommsSetup};

type LogMessage = (u64, TimelySetup, TimelyEvent);
type CommsMessage = (u64, CommsSetup, CommsEvent);

/// A log writer.
pub type Logger = Rc<BufferingLogger<TimelySetup, TimelyEvent>>;

/// A log writer that does not log anything.
pub fn new_inactive_logger() -> Logger {
    BufferingLogger::<TimelySetup, TimelyEvent>::new_inactive()
}

/// Shared wrapper for log writer constructors.
pub struct LoggerConfig {
    /// Log writer constructors.
    pub timely_logging: Arc<Fn(TimelySetup)->Rc<BufferingLogger<TimelySetup, TimelyEvent>>+Send+Sync>,
    /// Log writer constructors for communication.
    pub communication_logging: Arc<Fn(CommsSetup)->Rc<BufferingLogger<CommsSetup, CommsEvent>>+Send+Sync>,
}

impl LoggerConfig {
    /// Makes a new `LoggerConfig` wrapper from a `LogManager`.
    pub fn new<P1: 'static, P2: 'static, F1: 'static, F2: 'static>(
        timely_subscription: F1, communication_subscription: F2) -> Self where
        P1: EventPusher<Product<RootTimestamp, u64>, LogMessage> + Send,
        P2: EventPusher<Product<RootTimestamp, u64>, CommsMessage> + Send,
        F1: Fn(TimelySetup)->P1+Send+Sync,
        F2: Fn(CommsSetup)->P2+Send+Sync {

        LoggerConfig {
            timely_logging: Arc::new(move |events_setup: TimelySetup| {
                let logger = RefCell::new(BatchLogger::new((timely_subscription)(events_setup)));
                Rc::new(BufferingLogger::new(events_setup, Box::new(move |data| logger.borrow_mut().publish_batch(data))))
            }),
            communication_logging: Arc::new(move |comms_setup: CommsSetup| {
                let logger = RefCell::new(BatchLogger::new((communication_subscription)(comms_setup)));
                Rc::new(BufferingLogger::new(comms_setup, Box::new(move |data| logger.borrow_mut().publish_batch(data))))
            }),
        }
    }
}

impl Default for LoggerConfig {
    fn default() -> Self {
        LoggerConfig {
            timely_logging: Arc::new(|_setup| BufferingLogger::new_inactive()),
            communication_logging: Arc::new(|_setup| BufferingLogger::new_inactive()),
        }
    }
}

struct BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    // None when the logging stream is closed
    frontier: Option<Product<RootTimestamp, u64>>,
    event_pusher: P,
    _s: ::std::marker::PhantomData<S>,
    _e: ::std::marker::PhantomData<E>,
}

impl<S, E, P> BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    fn new(event_pusher: P) -> Self {
        BatchLogger {
            frontier: Some(Default::default()),
            event_pusher,
            _s: ::std::marker::PhantomData,
            _e: ::std::marker::PhantomData,
        }
    }
}

impl<S: Clone, E: Clone, P> BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    pub fn publish_batch(&mut self, logger_batch: LoggerBatch<S, E>) -> () {
        match logger_batch {
            LoggerBatch::Logs(evs) => {
                if let Some(frontier) = self.frontier {
                    let &(last_ts, _, _) = evs.last().unwrap();
                    self.event_pusher.push(Event::Messages(frontier, evs));
                    let new_frontier = RootTimestamp::new(last_ts);
                    self.event_pusher.push(Event::Progress(vec![(new_frontier, 1), (frontier, -1)]));
                    self.frontier = Some(new_frontier);
                }
            },
            LoggerBatch::End => {
                if let Some(frontier) = self.frontier {
                    self.event_pusher.push(Event::Progress(vec![(frontier, -1)]));
                    self.frontier = None;
                }
            },
        }
    }
}

/// An `EventPusher` that supports dynamically adding new `EventPushers`.
///
/// The tee maintains the frontier as the stream of events passes by. When a new pusher
/// arrives it advances the frontier to the current value, and starts to forward events
/// to it as well.
pub struct EventPusherTee<T: ::order::PartialOrder+Ord+Default+Clone+'static, D: Clone> {
    frontier: MutableAntichain<T>,
    listeners: Vec<Box<EventPusher<T, D>+Send>>,
}

impl<T: ::order::PartialOrder+Ord+Default+Clone+'static, D: Clone> EventPusherTee<T, D> {
    /// Construct a new tee with no subscribers.
    pub fn new() -> Self {
        Self {
            frontier: Default::default(),
            listeners: Vec::new(),
        }
    }
    /// Subscribe to this tee.
    pub fn subscribe(&mut self, mut listener: Box<EventPusher<T, D>+Send>) {
        let mut changes = vec![(Default::default(), -1)];
        changes.extend(self.frontier.frontier().iter().map(|x| (x.clone(), 1)));
        listener.push(Event::Progress(changes));
        self.listeners.push(listener);
    }
}

impl<T: ::order::PartialOrder+Ord+Default+Clone, D: Clone> EventPusher<T, D> for EventPusherTee<T, D> {
    fn push(&mut self, event: Event<T, D>) {
        // update the maintained frontier.
        if let Event::Progress(ref updates) = event {
            self.frontier.update_iter(updates.iter().cloned());
        }
        // present the event to each listener.
        for listener in self.listeners.iter_mut() {
            listener.push(event.clone());
        }
    }
}

#[derive(Abomonation, Debug, Clone)]
/// The creation of an `Operate` implementor.
pub struct OperatesEvent {
    /// Worker-unique identifier for the operator.
    pub id: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub addr: Vec<usize>,
    /// A helpful name.
    pub name: String,
}

#[derive(Abomonation, Debug, Clone)]
/// The creation of a channel between operators.
pub struct ChannelsEvent {
    /// Worker-unique identifier for the channel
    pub id: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub scope_addr: Vec<usize>,
    /// Source descriptor, indicating operator index and output port.
    pub source: (usize, usize),
    /// Target descriptor, indicating operator index and input port.
    pub target: (usize, usize),
}

#[derive(Abomonation, Debug, Clone)]
/// Send or receive of progress information.
pub struct ProgressEvent {
    /// `true` if the event is a send, and `false` if it is a receive.
    pub is_send: bool,
    /// Source worker index.
    pub source: usize,
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Message sequence number.
    pub seq_no: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub addr: Vec<usize>,
    /// List of message updates, containing Target descriptor, timestamp as string, and delta.
    pub messages: Vec<(usize, usize, String, i64)>,
    /// List of capability updates, containing Source descriptor, timestamp as string, and delta.
    pub internal: Vec<(usize, usize, String, i64)>,
}

#[derive(Abomonation, Debug, Clone)]
/// External progress pushed onto an operator
pub struct PushProgressEvent {
    /// Worker-unique operator identifier
    pub op_id: usize,
}

#[derive(Abomonation, Debug, Clone)]
/// Message send or receive event
pub struct MessagesEvent {
    /// `true` if send event, `false` if receive event.
    pub is_send: bool,
    /// Channel identifier
    pub channel: usize,
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Source worker index.
    pub source: usize,
    /// Target worker index.
    pub target: usize,
    /// Message sequence number.
    pub seq_no: usize,
    /// Number of typed records in the message.
    pub length: usize,
}

/// Records the starting and stopping of an operator.
#[derive(Abomonation, Debug, Clone, PartialEq, Eq)]
pub enum StartStop {
    /// Operator starts.
    Start,
    /// Operator stops; did it have any activity?
    Stop {
        /// Did the operator perform non-trivial work.
        activity: bool
    },
}

#[derive(Abomonation, Debug, Clone)]
/// Operator start or stop.
pub struct ScheduleEvent {
    /// Worker-unique identifier for the operator, linkable to the identifiers in `OperatesEvent`.
    pub id: usize,
    /// `Start` if the operator is starting, `Stop` if it is stopping.
    /// activity is true if it looks like some useful work was performed during this call (data was
    /// read or written, notifications were requested / delivered)
    pub start_stop: StartStop,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code startor stop
pub struct ApplicationEvent {
    /// Unique event type identifier
    pub id: usize,
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code start or stop
pub struct GuardedMessageEvent {
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code start or stop
pub struct GuardedProgressEvent {
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, PartialEq, Eq, Hash, Clone, Copy)]
/// Identifier of the worker that generated a log line
pub struct TimelySetup {
    /// Worker index
    pub index: usize,
}

#[derive(Abomonation, Debug, Clone)]
/// Kind of communication channel
pub enum CommChannelKind {
    /// Communication channel carrying progress information
    Progress,
    /// Communication channel carrying data
    Data,
}

#[derive(Abomonation, Debug, Clone)]
/// Event on a communication channel
pub struct CommChannelsEvent {
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Kind of communication channel (progress / data)
    pub comm_channel_kind: CommChannelKind,
}

#[derive(Abomonation, Debug, Clone)]
/// Input logic start/stop
pub struct InputEvent {
    /// True when activity begins, false when it stops
    pub start_stop: StartStop,
}

#[derive(Debug, Clone, Abomonation)]
/// An event in a timely worker
#[allow(missing_docs)]
pub enum TimelyEvent {
    /*  0 */ Operates(OperatesEvent),
    /*  1 */ Channels(ChannelsEvent),
    /*  2 */ Progress(ProgressEvent),
    /*  3 */ PushProgress(PushProgressEvent),
    /*  4 */ Messages(MessagesEvent),
    /*  5 */ Schedule(ScheduleEvent),
    /*  6 */ Application(ApplicationEvent),
    /*  7 */ GuardedMessage(GuardedMessageEvent),
    /*  8 */ GuardedProgress(GuardedProgressEvent),
    /*  9 */ CommChannels(CommChannelsEvent),
    /* 10 */ Input(InputEvent),
}

impl From<OperatesEvent> for TimelyEvent {
    fn from(v: OperatesEvent) -> TimelyEvent { TimelyEvent::Operates(v) }
}

impl From<ChannelsEvent> for TimelyEvent {
    fn from(v: ChannelsEvent) -> TimelyEvent { TimelyEvent::Channels(v) }
}

impl From<ProgressEvent> for TimelyEvent {
    fn from(v: ProgressEvent) -> TimelyEvent { TimelyEvent::Progress(v) }
}

impl From<PushProgressEvent> for TimelyEvent {
    fn from(v: PushProgressEvent) -> TimelyEvent { TimelyEvent::PushProgress(v) }
}

impl From<MessagesEvent> for TimelyEvent {
    fn from(v: MessagesEvent) -> TimelyEvent { TimelyEvent::Messages(v) }
}

impl From<ScheduleEvent> for TimelyEvent {
    fn from(v: ScheduleEvent) -> TimelyEvent { TimelyEvent::Schedule(v) }
}

impl From<ApplicationEvent> for TimelyEvent {
    fn from(v: ApplicationEvent) -> TimelyEvent { TimelyEvent::Application(v) }
}

impl From<GuardedMessageEvent> for TimelyEvent {
    fn from(v: GuardedMessageEvent) -> TimelyEvent { TimelyEvent::GuardedMessage(v) }
}

impl From<GuardedProgressEvent> for TimelyEvent {
    fn from(v: GuardedProgressEvent) -> TimelyEvent { TimelyEvent::GuardedProgress(v) }
}

impl From<CommChannelsEvent> for TimelyEvent {
    fn from(v: CommChannelsEvent) -> TimelyEvent { TimelyEvent::CommChannels(v) }
}

impl From<InputEvent> for TimelyEvent {
    fn from(v: InputEvent) -> TimelyEvent { TimelyEvent::Input(v) }
}
<commit_msg>logging: Start with a non-empty frontier in EventPusherTee<commit_after>//! Traits, implementations, and macros related to logging timely events.

use std::rc::Rc;
use std::cell::RefCell;
use std::sync::Arc;

use ::progress::change_batch::ChangeBatch;
use ::progress::timestamp::RootTimestamp;
use ::progress::nested::product::Product;
use ::progress::frontier::MutableAntichain;

use dataflow::operators::capture::{Event, EventPusher};

use timely_communication::logging::{BufferingLogger, LoggerBatch, CommsEvent, CommsSetup};

type LogMessage = (u64, TimelySetup, TimelyEvent);
type CommsMessage = (u64, CommsSetup, CommsEvent);

/// A log writer.
pub type Logger = Rc<BufferingLogger<TimelySetup, TimelyEvent>>;

/// A log writer that does not log anything.
pub fn new_inactive_logger() -> Logger {
    BufferingLogger::<TimelySetup, TimelyEvent>::new_inactive()
}

/// Shared wrapper for log writer constructors.
pub struct LoggerConfig {
    /// Log writer constructors.
    pub timely_logging: Arc<Fn(TimelySetup)->Rc<BufferingLogger<TimelySetup, TimelyEvent>>+Send+Sync>,
    /// Log writer constructors for communication.
    pub communication_logging: Arc<Fn(CommsSetup)->Rc<BufferingLogger<CommsSetup, CommsEvent>>+Send+Sync>,
}

impl LoggerConfig {
    /// Makes a new `LoggerConfig` wrapper from a `LogManager`.
    pub fn new<P1: 'static, P2: 'static, F1: 'static, F2: 'static>(
        timely_subscription: F1, communication_subscription: F2) -> Self where
        P1: EventPusher<Product<RootTimestamp, u64>, LogMessage> + Send,
        P2: EventPusher<Product<RootTimestamp, u64>, CommsMessage> + Send,
        F1: Fn(TimelySetup)->P1+Send+Sync,
        F2: Fn(CommsSetup)->P2+Send+Sync {

        LoggerConfig {
            timely_logging: Arc::new(move |events_setup: TimelySetup| {
                let logger = RefCell::new(BatchLogger::new((timely_subscription)(events_setup)));
                Rc::new(BufferingLogger::new(events_setup, Box::new(move |data| logger.borrow_mut().publish_batch(data))))
            }),
            communication_logging: Arc::new(move |comms_setup: CommsSetup| {
                let logger = RefCell::new(BatchLogger::new((communication_subscription)(comms_setup)));
                Rc::new(BufferingLogger::new(comms_setup, Box::new(move |data| logger.borrow_mut().publish_batch(data))))
            }),
        }
    }
}

impl Default for LoggerConfig {
    fn default() -> Self {
        LoggerConfig {
            timely_logging: Arc::new(|_setup| BufferingLogger::new_inactive()),
            communication_logging: Arc::new(|_setup| BufferingLogger::new_inactive()),
        }
    }
}

struct BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    // None when the logging stream is closed
    frontier: Option<Product<RootTimestamp, u64>>,
    event_pusher: P,
    _s: ::std::marker::PhantomData<S>,
    _e: ::std::marker::PhantomData<E>,
}

impl<S, E, P> BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    fn new(event_pusher: P) -> Self {
        BatchLogger {
            frontier: Some(Default::default()),
            event_pusher,
            _s: ::std::marker::PhantomData,
            _e: ::std::marker::PhantomData,
        }
    }
}

impl<S: Clone, E: Clone, P> BatchLogger<S, E, P> where P: EventPusher<Product<RootTimestamp, u64>, (u64, S, E)> {
    pub fn publish_batch(&mut self, logger_batch: LoggerBatch<S, E>) -> () {
        match logger_batch {
            LoggerBatch::Logs(evs) => {
                if let Some(frontier) = self.frontier {
                    let &(last_ts, _, _) = evs.last().unwrap();
                    self.event_pusher.push(Event::Messages(frontier, evs));
                    let new_frontier = RootTimestamp::new(last_ts);
                    self.event_pusher.push(Event::Progress(vec![(new_frontier, 1), (frontier, -1)]));
                    self.frontier = Some(new_frontier);
                }
            },
            LoggerBatch::End => {
                if let Some(frontier) = self.frontier {
                    self.event_pusher.push(Event::Progress(vec![(frontier, -1)]));
                    self.frontier = None;
                }
            },
        }
    }
}

/// An `EventPusher` that supports dynamically adding new `EventPushers`.
///
/// The tee maintains the frontier as the stream of events passes by. When a new pusher
/// arrives it advances the frontier to the current value, and starts to forward events
/// to it as well.
pub struct EventPusherTee<T: ::order::PartialOrder+Ord+Default+Clone+'static, D: Clone> {
    frontier: MutableAntichain<T>,
    listeners: Vec<Box<EventPusher<T, D>+Send>>,
}

impl<T: ::order::PartialOrder+Ord+Default+Clone+'static, D: Clone> EventPusherTee<T, D> {
    /// Construct a new tee with no subscribers.
    pub fn new() -> Self {
        Self {
            frontier: MutableAntichain::new_bottom(Default::default()),
            listeners: Vec::new(),
        }
    }
    /// Subscribe to this tee.
    pub fn subscribe(&mut self, mut listener: Box<EventPusher<T, D>+Send>) {
        let mut changes = ChangeBatch::new_from(Default::default(), -1);
        changes.extend(self.frontier.frontier().iter().map(|x| (x.clone(), 1)));
        if !changes.is_empty() {
            listener.push(Event::Progress(changes.into_inner()));
        }
        self.listeners.push(listener);
    }
}

impl<T: ::order::PartialOrder+Ord+Default+Clone, D: Clone> EventPusher<T, D> for EventPusherTee<T, D> {
    fn push(&mut self, event: Event<T, D>) {
        // update the maintained frontier.
        if let Event::Progress(ref updates) = event {
            self.frontier.update_iter(updates.iter().cloned());
        }
        // present the event to each listener.
        for listener in self.listeners.iter_mut() {
            listener.push(event.clone());
        }
    }
}

#[derive(Abomonation, Debug, Clone)]
/// The creation of an `Operate` implementor.
pub struct OperatesEvent {
    /// Worker-unique identifier for the operator.
    pub id: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub addr: Vec<usize>,
    /// A helpful name.
    pub name: String,
}

#[derive(Abomonation, Debug, Clone)]
/// The creation of a channel between operators.
pub struct ChannelsEvent {
    /// Worker-unique identifier for the channel
    pub id: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub scope_addr: Vec<usize>,
    /// Source descriptor, indicating operator index and output port.
    pub source: (usize, usize),
    /// Target descriptor, indicating operator index and input port.
    pub target: (usize, usize),
}

#[derive(Abomonation, Debug, Clone)]
/// Send or receive of progress information.
pub struct ProgressEvent {
    /// `true` if the event is a send, and `false` if it is a receive.
    pub is_send: bool,
    /// Source worker index.
    pub source: usize,
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Message sequence number.
    pub seq_no: usize,
    /// Sequence of nested scope identifiers indicating the path from the root to this instance.
    pub addr: Vec<usize>,
    /// List of message updates, containing Target descriptor, timestamp as string, and delta.
    pub messages: Vec<(usize, usize, String, i64)>,
    /// List of capability updates, containing Source descriptor, timestamp as string, and delta.
    pub internal: Vec<(usize, usize, String, i64)>,
}

#[derive(Abomonation, Debug, Clone)]
/// External progress pushed onto an operator
pub struct PushProgressEvent {
    /// Worker-unique operator identifier
    pub op_id: usize,
}

#[derive(Abomonation, Debug, Clone)]
/// Message send or receive event
pub struct MessagesEvent {
    /// `true` if send event, `false` if receive event.
    pub is_send: bool,
    /// Channel identifier
    pub channel: usize,
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Source worker index.
    pub source: usize,
    /// Target worker index.
    pub target: usize,
    /// Message sequence number.
    pub seq_no: usize,
    /// Number of typed records in the message.
    pub length: usize,
}

/// Records the starting and stopping of an operator.
#[derive(Abomonation, Debug, Clone, PartialEq, Eq)]
pub enum StartStop {
    /// Operator starts.
    Start,
    /// Operator stops; did it have any activity?
    Stop {
        /// Did the operator perform non-trivial work.
        activity: bool
    },
}

#[derive(Abomonation, Debug, Clone)]
/// Operator start or stop.
pub struct ScheduleEvent {
    /// Worker-unique identifier for the operator, linkable to the identifiers in `OperatesEvent`.
    pub id: usize,
    /// `Start` if the operator is starting, `Stop` if it is stopping.
    /// activity is true if it looks like some useful work was performed during this call (data was
    /// read or written, notifications were requested / delivered)
    pub start_stop: StartStop,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code startor stop
pub struct ApplicationEvent {
    /// Unique event type identifier
    pub id: usize,
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code start or stop
pub struct GuardedMessageEvent {
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, Clone)]
/// Application-defined code start or stop
pub struct GuardedProgressEvent {
    /// True when activity begins, false when it stops
    pub is_start: bool,
}

#[derive(Abomonation, Debug, PartialEq, Eq, Hash, Clone, Copy)]
/// Identifier of the worker that generated a log line
pub struct TimelySetup {
    /// Worker index
    pub index: usize,
}

#[derive(Abomonation, Debug, Clone)]
/// Kind of communication channel
pub enum CommChannelKind {
    /// Communication channel carrying progress information
    Progress,
    /// Communication channel carrying data
    Data,
}

#[derive(Abomonation, Debug, Clone)]
/// Event on a communication channel
pub struct CommChannelsEvent {
    /// Communication channel identifier
    pub comm_channel: Option<usize>,
    /// Kind of communication channel (progress / data)
    pub comm_channel_kind: CommChannelKind,
}

#[derive(Abomonation, Debug, Clone)]
/// Input logic start/stop
pub struct InputEvent {
    /// True when activity begins, false when it stops
    pub start_stop: StartStop,
}

#[derive(Debug, Clone, Abomonation)]
/// An event in a timely worker
#[allow(missing_docs)]
pub enum TimelyEvent {
    /*  0 */ Operates(OperatesEvent),
    /*  1 */ Channels(ChannelsEvent),
    /*  2 */ Progress(ProgressEvent),
    /*  3 */ PushProgress(PushProgressEvent),
    /*  4 */ Messages(MessagesEvent),
    /*  5 */ Schedule(ScheduleEvent),
    /*  6 */ Application(ApplicationEvent),
    /*  7 */ GuardedMessage(GuardedMessageEvent),
    /*  8 */ GuardedProgress(GuardedProgressEvent),
    /*  9 */ CommChannels(CommChannelsEvent),
    /* 10 */ Input(InputEvent),
}

impl From<OperatesEvent> for TimelyEvent {
    fn from(v: OperatesEvent) -> TimelyEvent { TimelyEvent::Operates(v) }
}

impl From<ChannelsEvent> for TimelyEvent {
    fn from(v: ChannelsEvent) -> TimelyEvent { TimelyEvent::Channels(v) }
}

impl From<ProgressEvent> for TimelyEvent {
    fn from(v: ProgressEvent) -> TimelyEvent { TimelyEvent::Progress(v) }
}

impl From<PushProgressEvent> for TimelyEvent {
    fn from(v: PushProgressEvent) -> TimelyEvent { TimelyEvent::PushProgress(v) }
}

impl From<MessagesEvent> for TimelyEvent {
    fn from(v: MessagesEvent) -> TimelyEvent { TimelyEvent::Messages(v) }
}

impl From<ScheduleEvent> for TimelyEvent {
    fn from(v: ScheduleEvent) -> TimelyEvent { TimelyEvent::Schedule(v) }
}

impl From<ApplicationEvent> for TimelyEvent {
    fn from(v: ApplicationEvent) -> TimelyEvent { TimelyEvent::Application(v) }
}

impl From<GuardedMessageEvent> for TimelyEvent {
    fn from(v: GuardedMessageEvent) -> TimelyEvent { TimelyEvent::GuardedMessage(v) }
}

impl From<GuardedProgressEvent> for TimelyEvent {
    fn from(v: GuardedProgressEvent) -> TimelyEvent { TimelyEvent::GuardedProgress(v) }
}

impl From<CommChannelsEvent> for TimelyEvent {
    fn from(v: CommChannelsEvent) -> TimelyEvent { TimelyEvent::CommChannels(v) }
}

impl From<InputEvent> for TimelyEvent {
    fn from(v: InputEvent) -> TimelyEvent { TimelyEvent::Input(v) }
}
<|endoftext|>


<commit_before>//! Lua functionality

use hlua;
use hlua::{Lua, LuaError};
use hlua::any::AnyLuaValue;

use std::thread;

use std::fs::{File};
use std::path::Path;
use std::io::Write;

use std::sync::{Mutex, RwLock};
use std::sync::mpsc::{channel, Sender, Receiver};

#[macro_use]
mod funcs;

lazy_static! {
    /// Sends requests to the lua thread
    static ref SENDER: Mutex<Option<Sender<LuaQuery>>> = Mutex::new(None);

    /// Receives data back from the lua thread
    /// This should only be accessed by the lua thread itself.
    static ref RECEIVER: Mutex<Option<Receiver<LuaResponse>>> = Mutex::new(None);

    /// Whether the lua thread is currently running
    pub static ref RUNNING: RwLock<bool> = RwLock::new(false);
}

/// Messages sent to the lua thread
pub enum LuaQuery {
    /// Halt the lua thread
    Terminate,
    // Restart the lua thread
    Restart,
    /// Execute a string
    Execute(String),
    /// Execute a file
    ExecuteFile(String),
    /// Get a variable
    GetVariable(String),
    /// Set a value
    SetValue {
        name: Box<::std::borrow::Borrow<str> + Sized>,
        val: Box<hlua::Push<&'static mut Lua<'static>> + Sized>
    },
    /// Create a new array
    EmptyArray(String),
    /// Message to ping the lua thread
    Ping,
    /// Unused send type
    Unused,
}

/// Messages received from lua thread
pub enum LuaResponse {
    /// Lua variable obtained
    Variable(Option<AnyLuaValue>),
    /// Lua error
    Error(hlua::LuaError),
    /// A function is returned
    Function(hlua::functions_read::LuaFunction<String>),
    /// Pong response from lua ping
    Pong,
    /// Unused response type
    Unused,
}

unsafe impl Send for LuaQuery { }
unsafe impl Send for LuaResponse { }
unsafe impl Sync for LuaQuery { }
unsafe impl Sync for LuaResponse { }

/// Whether the lua thread is currently available
pub fn thread_running() -> bool {
    *RUNNING.read().unwrap()
}

/// Errors which may arise from attempting
/// to sending a message to the lua thread.
#[derive(Debug)]
pub enum LuaSendError {
    ThreadClosed,
    ThreadUninitialized,
    Sender
}

/// Attemps to send a LuaQuery to the lua thread.
pub fn try_send(query: LuaQuery) -> Result<(), LuaSendError> {
    if !thread_running() {
        Err(LuaSendError::ThreadClosed)
    }
    else if let Some(ref sender) = *SENDER.lock().unwrap() {
        match sender.send(query) {
            Ok(_) => Ok(()),
            Err(_) => Err(LuaSendError::Sender)
        }
    }
    else {
        Err(LuaSendError::ThreadUninitialized)
    }
}

/// Initialize the lua thread
pub fn init() {
    trace!("Initializing...");
    let (query_tx, query_rx) = channel::<LuaQuery>();
    let (answer_tx, answer_rx) = channel::<LuaResponse>();
    {
        let mut sender = SENDER.lock().unwrap();
        let mut receiver = RECEIVER.lock().unwrap();

        *sender = Some(query_tx);
        *receiver = Some(answer_rx);
    }

    thread::spawn(move || {
        thread_init(answer_tx, query_rx);
    });
    trace!("Created thread. Init finished.");
}

fn thread_init(sender: Sender<LuaResponse>, receiver: Receiver<LuaQuery>) {
    trace!("thread: initializing.");
    let mut lua = Lua::new();
    debug!("thread: Loading Lua libraries...");
    lua.openlibs();
    trace!("thread: Loading way-cooler lua extensions...");
    // We should have some good file handling, read files from /usr by default,
    // but for now we're reading directly from the source.
    lua.execute_from_reader::<(), File>(
        File::open("lib/lua/init.lua").unwrap()
    );
    trace!("thread: loading way-cooler libraries...");
    funcs::register_libraries(&mut lua);
    // Only ready after loading libs
    *RUNNING.write().unwrap() = true;
    debug!("thread: entering main loop...");
    thread_main_loop(sender, receiver, &mut lua);
}

fn thread_main_loop(sender: Sender<LuaResponse>, receiver: Receiver<LuaQuery>,
                    lua: &mut Lua) {
    loop {
        let request = receiver.recv();
        match request {
            Err(e) => {
                error!("Lua thread: unable to receive message: {}", e);
                error!("Lua thread: now panicking!");
                *RUNNING.write().unwrap() = false;

                panic!("Lua thread: lost contact with host, exiting!");
            }
            Ok(message) => {
                trace!("Handling a request");
                thread_handle_message(&sender, message, lua);
            }
        }
    }
}

fn thread_handle_message(sender: &Sender<LuaResponse>,
                         request: LuaQuery, lua: &mut Lua) {
    match request {
        LuaQuery::Terminate => {
            trace!("thread: Received terminate signal");
            *RUNNING.write().unwrap() = false;

            info!("thread: Lua thread terminating!");
            return;
        },

        LuaQuery::Restart => {
            trace!("thread: Received restart signal!");
            error!("thread: Lua thread restart not supported!");

            *RUNNING.write().unwrap() = false;

            panic!("Lua thread: Restart not supported!");
        },

        LuaQuery::Execute(code) => {
            trace!("thread: Received request to execute code");
            trace!("thread: Executing {:?}", code);

            match lua.execute::<()>(&code) {
                Err(error) => {
                    warn!("thread: Error executing code: {:?}", error);
                    let response = LuaResponse::Error(error);

                    thread_send(&sender, response);
                }
                Ok(_) => {
                    // This is gonna be really spammy one day
                    trace!("thread: Code executed okay.");
                }
            }
        },

        LuaQuery::ExecuteFile(name) => {
            trace!("thread: Received request to execute file {}", name);
            info!("thread: Executing {}", name);

            let path = Path::new(&name);
            let try_file = File::open(path);

            if let Ok(file) = try_file {
                let result = lua.execute_from_reader::<(), File>(file);
                if let Err(err) = result {
                    warn!("thread: Error executing {}!", name);

                    thread_send(&sender, LuaResponse::Error(err));
                }
                else {
                    trace!("thread: Execution of {} successful.", name);
                }
            }
            else { // Could not open file
                // Unwrap_err is used because we're in the else of let
                let read_error =
                    LuaError::ReadError(try_file.unwrap_err());

                thread_send(&sender, LuaResponse::Error(read_error));
            }
        },

        LuaQuery::GetVariable(varname) => {
            trace!("thread: Received request to get variable {}", varname);
            let var_result = lua.get(varname.as_str());

            match var_result {
                Some(var) => {
                    thread_send(&sender, LuaResponse::Variable(Some(var)));
                }
                None => {
                    warn!("thread: Unable to get variable {}", varname);

                    thread_send(&sender, LuaResponse::Variable(None));
                }
            }
        },

        LuaQuery::SetValue { name: name, val: val } => {
            panic!("thread: unimplemented LuaQuery::SetValue!");
        },

        LuaQuery::EmptyArray(name) => {
            panic!("thread: unimplemented LuaQuery::EmptyArray!");
        },

        _ => {
            panic!("Unimplemented send type for lua thread!");
        }
    }
}

fn thread_send(sender: &Sender<LuaResponse>, response: LuaResponse) {
    trace!("Called thread_send");
    match sender.send(response) {
        Err(e) => {
            error!("thread: Unable to broadcast response!");
            error!("thread: Shutting down in response to inability \
                    to continue!");
            panic!("Lua thread unable to communicate with main thread, \
                    shutting down!");
        }
        Ok(_) => {}
    }
}
<commit_msg>Attempt to add lua thread panic handler<commit_after>//! Lua functionality

use hlua;
use hlua::{Lua, LuaError, Push, PushGuard, AsMutLua, LuaContext};
use hlua::any::AnyLuaValue;

use hlua_ffi;
use hlua_ffi::lua_State;
use libc::c_int;

use std::thread;

use std::fs::{File};
use std::path::Path;
use std::io::Write;

use std::sync::{Mutex, RwLock};
use std::sync::mpsc::{channel, Sender, Receiver};

#[macro_use]
mod funcs;

lazy_static! {
    /// Sends requests to the lua thread
    static ref SENDER: Mutex<Option<Sender<LuaQuery>>> = Mutex::new(None);

    /// Receives data back from the lua thread
    /// This should only be accessed by the lua thread itself.
    static ref RECEIVER: Mutex<Option<Receiver<LuaResponse>>> = Mutex::new(None);

    /// Whether the lua thread is currently running
    pub static ref RUNNING: RwLock<bool> = RwLock::new(false);
}

/// Messages sent to the lua thread
pub enum LuaQuery {
    /// Halt the lua thread
    Terminate,
    // Restart the lua thread
    Restart,
    /// Execute a string
    Execute(String),
    /// Execute a file
    ExecuteFile(String),
    /// Get a variable
    GetVariable(String),
    /// Set a value
    SetValue {
        name: Box<::std::borrow::Borrow<str> + Sized>,
        val: Box<hlua::Push<&'static mut Lua<'static>> + Sized>
    },
    /// Create a new array
    EmptyArray(String),
    /// Message to ping the lua thread
    Ping,
    /// Unused send type
    Unused,
}

/// Messages received from lua thread
pub enum LuaResponse {
    /// Lua variable obtained
    Variable(Option<AnyLuaValue>),
    /// Lua error
    Error(hlua::LuaError),
    /// A function is returned
    Function(hlua::functions_read::LuaFunction<String>),
    /// Pong response from lua ping
    Pong,
    /// Unused response type
    Unused,
}

unsafe impl Send for LuaQuery { }
unsafe impl Send for LuaResponse { }
unsafe impl Sync for LuaQuery { }
unsafe impl Sync for LuaResponse { }

/// Whether the lua thread is currently available
pub fn thread_running() -> bool {
    *RUNNING.read().unwrap()
}

/// Errors which may arise from attempting
/// to sending a message to the lua thread.
#[derive(Debug)]
pub enum LuaSendError {
    ThreadClosed,
    ThreadUninitialized,
    Sender
}

/// Attemps to send a LuaQuery to the lua thread.
pub fn try_send(query: LuaQuery) -> Result<(), LuaSendError> {
    if !thread_running() {
        Err(LuaSendError::ThreadClosed)
    }
    else if let Some(ref sender) = *SENDER.lock().unwrap() {
        match sender.send(query) {
            Ok(_) => Ok(()),
            Err(_) => Err(LuaSendError::Sender)
        }
    }
    else {
        Err(LuaSendError::ThreadUninitialized)
    }
}

/// Initialize the lua thread
pub fn init() {
    trace!("Initializing...");
    let (query_tx, query_rx) = channel::<LuaQuery>();
    let (answer_tx, answer_rx) = channel::<LuaResponse>();
    {
        let mut sender = SENDER.lock().unwrap();
        let mut receiver = RECEIVER.lock().unwrap();

        *sender = Some(query_tx);
        *receiver = Some(answer_rx);
    }

    thread::spawn(move || {
        thread_init(answer_tx, query_rx);
    });
    trace!("Created thread. Init finished.");
}

fn thread_init(sender: Sender<LuaResponse>, receiver: Receiver<LuaQuery>) {
    trace!("thread: initializing.");
    let mut lua = Lua::new();
    //unsafe {
    //    hlua_ffi::lua_atpanic(&mut lua.as_mut_lua().0, thread_on_panic);
    //}
    debug!("thread: Loading Lua libraries...");
    lua.openlibs();
    trace!("thread: Loading way-cooler lua extensions...");
    // We should have some good file handling, read files from /usr by default,
    // but for now we're reading directly from the source.
    lua.execute_from_reader::<(), File>(
        File::open("lib/lua/init.lua").unwrap()
    );
    trace!("thread: loading way-cooler libraries...");
    funcs::register_libraries(&mut lua);
    // Only ready after loading libs
    *RUNNING.write().unwrap() = true;
    debug!("thread: entering main loop...");
    thread_main_loop(sender, receiver, &mut lua);
}

fn thread_main_loop(sender: Sender<LuaResponse>, receiver: Receiver<LuaQuery>,
                    lua: &mut Lua) {
    loop {
        let request = receiver.recv();
        match request {
            Err(e) => {
                error!("Lua thread: unable to receive message: {}", e);
                error!("Lua thread: now panicking!");
                *RUNNING.write().unwrap() = false;

                panic!("Lua thread: lost contact with host, exiting!");
            }
            Ok(message) => {
                trace!("Handling a request");
                thread_handle_message(&sender, message, lua);
            }
        }
    }
}

fn thread_handle_message(sender: &Sender<LuaResponse>,
                         request: LuaQuery, lua: &mut Lua) {
    match request {
        LuaQuery::Terminate => {
            trace!("thread: Received terminate signal");
            *RUNNING.write().unwrap() = false;

            info!("thread: Lua thread terminating!");
            return;
        },

        LuaQuery::Restart => {
            trace!("thread: Received restart signal!");
            error!("thread: Lua thread restart not supported!");

            *RUNNING.write().unwrap() = false;

            panic!("Lua thread: Restart not supported!");
        },

        LuaQuery::Execute(code) => {
            trace!("thread: Received request to execute code");
            trace!("thread: Executing {:?}", code);

            match lua.execute::<()>(&code) {
                Err(error) => {
                    warn!("thread: Error executing code: {:?}", error);
                    let response = LuaResponse::Error(error);

                    thread_send(&sender, response);
                }
                Ok(_) => {
                    // This is gonna be really spammy one day
                    trace!("thread: Code executed okay.");
                }
            }
        },

        LuaQuery::ExecuteFile(name) => {
            trace!("thread: Received request to execute file {}", name);
            info!("thread: Executing {}", name);

            let path = Path::new(&name);
            let try_file = File::open(path);

            if let Ok(file) = try_file {
                let result = lua.execute_from_reader::<(), File>(file);
                if let Err(err) = result {
                    warn!("thread: Error executing {}!", name);

                    thread_send(&sender, LuaResponse::Error(err));
                }
                else {
                    trace!("thread: Execution of {} successful.", name);
                }
            }
            else { // Could not open file
                // Unwrap_err is used because we're in the else of let
                let read_error =
                    LuaError::ReadError(try_file.unwrap_err());

                thread_send(&sender, LuaResponse::Error(read_error));
            }
        },

        LuaQuery::GetVariable(varname) => {
            trace!("thread: Received request to get variable {}", varname);
            let var_result = lua.get(varname.as_str());

            match var_result {
                Some(var) => {
                    thread_send(&sender, LuaResponse::Variable(Some(var)));
                }
                None => {
                    warn!("thread: Unable to get variable {}", varname);

                    thread_send(&sender, LuaResponse::Variable(None));
                }
            }
        },

        LuaQuery::SetValue { name: name, val: val } => {
            panic!("thread: unimplemented LuaQuery::SetValue!");
        },

        LuaQuery::EmptyArray(name) => {
            panic!("thread: unimplemented LuaQuery::EmptyArray!");
        },

        _ => {
            panic!("Unimplemented send type for lua thread!");
        }
    }
}

fn thread_send(sender: &Sender<LuaResponse>, response: LuaResponse) {
    trace!("Called thread_send");
    match sender.send(response) {
        Err(e) => {
            error!("thread: Unable to broadcast response!");
            error!("thread: Shutting down in response to inability \
                    to continue!");
            panic!("Lua thread unable to communicate with main thread, \
                    shutting down!");
        }
        Ok(_) => {}
    }
}

extern "C" fn thread_on_panic(state: *mut lua_State) -> c_int {
    *RUNNING.write().unwrap() = false;
    error!("Lua thread is panicking!");
    0
}
<|endoftext|>


<commit_before>use unix_socket::UnixStream;

use std::io;
use std::io::{Read, Write};

const LVMETAD_PATH: &'static str = "/run/lvm/lvmetad.socket";

fn response(stream: &mut UnixStream) -> io::Result<Vec<u8>> {
    let mut response = [0; 32];
    let mut v = Vec::new();

    loop {
        let bytes_read = try!(stream.read(&mut response));

        v.push_all(&response[..bytes_read]);

        if v.ends_with(b"\n##\n") {
            // drop the end marker
            let len = v.len() - 4;
            v.truncate(len);
            return Ok(v);
        }
    }
}

fn open_lvmetad() {

    request(b"hello", false);
//    lvmetad_request(b"vg_list", true);
//    lvmetad_request(b"pv_list", true);
//    lvmetad_request(b"dump", false);

}

fn request(s: &[u8], token: bool) -> io::Result<Vec<u8>>{

    let mut stream = UnixStream::connect(LVMETAD_PATH).unwrap();
    stream.write_all(b"request = \"").unwrap();
    stream.write_all(s).unwrap();
    stream.write_all(b"\"\n").unwrap();
    if token {
        stream.write_all(b"token = \"filter:0\"").unwrap();
        stream.write_all(b"\n").unwrap();
    }
    stream.write_all(b"\n##\n").unwrap();

    let response = try!(response(&mut stream));

    Ok(response)
}
<commit_msg>make request() pub<commit_after>use unix_socket::UnixStream;

use std::io;
use std::io::{Read, Write};

const LVMETAD_PATH: &'static str = "/run/lvm/lvmetad.socket";

fn response(stream: &mut UnixStream) -> io::Result<Vec<u8>> {
    let mut response = [0; 32];
    let mut v = Vec::new();

    loop {
        let bytes_read = try!(stream.read(&mut response));

        v.push_all(&response[..bytes_read]);

        if v.ends_with(b"\n##\n") {
            // drop the end marker
            let len = v.len() - 4;
            v.truncate(len);
            return Ok(v);
        }
    }
}

fn open_lvmetad() {

    request(b"hello", false);
//    lvmetad_request(b"vg_list", true);
//    lvmetad_request(b"pv_list", true);
//    lvmetad_request(b"dump", false);

}

pub fn request(s: &[u8], token: bool) -> io::Result<Vec<u8>>{

    let mut stream = UnixStream::connect(LVMETAD_PATH).unwrap();
    stream.write_all(b"request = \"").unwrap();
    stream.write_all(s).unwrap();
    stream.write_all(b"\"\n").unwrap();
    if token {
        stream.write_all(b"token = \"filter:0\"").unwrap();
        stream.write_all(b"\n").unwrap();
    }
    stream.write_all(b"\n##\n").unwrap();

    let response = try!(response(&mut stream));

    Ok(response)
}
<|endoftext|>


<commit_before>use std;
use std::fmt;
use std::net::TcpStream;
use rustc_serialize::{Encodable, Decodable};
use msgpack::{self, Encoder, Decoder};

pub enum JoinRequest {
    Broadcast(String),
    Watch(String),
}

pub enum JoinResponse {
    Success,
    Failure,
}

pub enum Notification {
    Output(String),
    Closed(String),
    WatcherJoined(String),
}

#[derive(Debug)]
pub enum Error {
    EncodeError(msgpack::encode::Error),
    DecodeError(msgpack::decode::Error),
    UnknownMessage,
}

impl JoinRequest {
    pub fn receive(stream: &TcpStream) -> Result<JoinRequest, Error> {
        let mut decoder = Decoder::new(stream);

        match Decodable::decode(&mut decoder) {
            Ok(message) => {
                let (_, _, role, channel_name): (u8, u8, String, String) = message;

                match role.as_ref() {
                    "broadcast" => Ok(JoinRequest::Broadcast(channel_name)),
                    "watch"     => Ok(JoinRequest::Watch(channel_name)),
                    _           => Err(Error::UnknownMessage)
                }

            },
            Err(e) => Err(Error::from(e))
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        self.payload().encode(&mut encoder).or_else(|e| Err(Error::from(e)))
    }

    fn payload(&self) -> (u8, u8, &str, &String) {
        let header = 0u8;
        let id     = 0u8;

        match *self {
            JoinRequest::Broadcast(ref string) => (header, id, "broadcast", string),
            JoinRequest::Watch(ref string)     => (header, id, "watch",     string),
        }
    }
}

impl JoinResponse {
    pub fn receive(stream: &TcpStream) -> Result<JoinResponse, Error> {
        let mut decoder = Decoder::new(stream);

        match Decodable::decode(&mut decoder) {
            Ok(message) => {
                let (_, _, _, result): (u8, u8, String, bool) = message;

                match result {
                    true  => Ok(JoinResponse::Success),
                    false => Ok(JoinResponse::Failure),
                }

            },
            Err(e) => Err(Error::from(e))
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        self.payload().encode(&mut encoder).or_else(|e| Err(Error::from(e)))
    }

    fn payload(&self) -> (u8, u8, &str, bool) {
        let header = 0u8;
        let id     = 0u8;

        match *self {
            JoinResponse::Success => (header, id, "", true),
            JoinResponse::Failure => (header, id, "", false),
        }
    }
}

impl Notification {
    pub fn receive(stream: &TcpStream) -> Result<Notification, Error> {
        let mut decoder = Decoder::new(stream);

        match Decodable::decode(&mut decoder) {
            Ok(message) => {
                let (_, topic, data): (u8, String, String) = message;

                match topic.as_ref() {
                    "out"            => Ok(Notification::Output(data)),
                    "closed"         => Ok(Notification::Closed(data)),
                    "watcher_joined" => Ok(Notification::WatcherJoined(data)),
                    _                => Err(Error::UnknownMessage)
                }

            },
            Err(e) => Err(Error::from(e))
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        self.payload().encode(&mut encoder).or_else(|e| Err(Error::from(e)))
    }

    fn payload(&self) -> (u8, &str, &String) {
        let header = 2u8;

        match *self {
            Notification::Output(ref string)        => (header, "out",            string),
            Notification::Closed(ref string)        => (header, "closed",         string),
            Notification::WatcherJoined(ref string) => (header, "watcher_joined", string),
        }
    }
}

impl std::error::Error for Error {
    fn description(&self) -> &str {
        "Processing message failed"
    }

    fn cause(&self) -> Option<&::std::error::Error> {
        match *self {
            Error::EncodeError(ref err) => Some(err),
            Error::DecodeError(ref err) => Some(err),
            _ => None
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        std::error::Error::description(self).fmt(f)
    }
}

impl From<msgpack::encode::Error> for Error {
    fn from(err: msgpack::encode::Error) -> Error {
        Error::EncodeError(err)
    }
}

impl From<msgpack::decode::Error> for Error {
    fn from(err: msgpack::decode::Error) -> Error {
        Error::DecodeError(err)
    }
}
<commit_msg>Make brevity by using `try!` instead of `match`<commit_after>use std;
use std::fmt;
use std::net::TcpStream;
use rustc_serialize::{Encodable, Decodable};
use msgpack::{self, Encoder, Decoder};

pub enum JoinRequest {
    Broadcast(String),
    Watch(String),
}

pub enum JoinResponse {
    Success,
    Failure,
}

pub enum Notification {
    Output(String),
    Closed(String),
    WatcherJoined(String),
}

#[derive(Debug)]
pub enum Error {
    EncodeError(msgpack::encode::Error),
    DecodeError(msgpack::decode::Error),
    UnknownMessage,
}

impl JoinRequest {
    pub fn receive(stream: &TcpStream) -> Result<JoinRequest, Error> {
        let (_, _, role, channel_name): (u8, u8, String, String) = try!(Decodable::decode(&mut Decoder::new(stream)));

        match role.as_ref() {
            "broadcast" => Ok(JoinRequest::Broadcast(channel_name)),
            "watch"     => Ok(JoinRequest::Watch(channel_name)),
            _           => Err(Error::UnknownMessage)
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        Ok(try!(self.payload().encode(&mut encoder)))
    }

    fn payload(&self) -> (u8, u8, &str, &String) {
        let header = 0u8;
        let id     = 0u8;

        match *self {
            JoinRequest::Broadcast(ref string) => (header, id, "broadcast", string),
            JoinRequest::Watch(ref string)     => (header, id, "watch",     string),
        }
    }
}

impl JoinResponse {
    pub fn receive(stream: &TcpStream) -> Result<JoinResponse, Error> {
        let (_, _, _, result): (u8, u8, String, bool) = try!(Decodable::decode(&mut Decoder::new(stream)));

        match result {
            true  => Ok(JoinResponse::Success),
            false => Ok(JoinResponse::Failure),
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        Ok(try!(self.payload().encode(&mut encoder)))
    }

    fn payload(&self) -> (u8, u8, &str, bool) {
        let header = 0u8;
        let id     = 0u8;

        match *self {
            JoinResponse::Success => (header, id, "", true),
            JoinResponse::Failure => (header, id, "", false),
        }
    }
}

impl Notification {
    pub fn receive(stream: &TcpStream) -> Result<Notification, Error> {
        let (_, topic, data): (u8, String, String) = try!(Decodable::decode(&mut Decoder::new(stream)));

        match topic.as_ref() {
            "out"            => Ok(Notification::Output(data)),
            "closed"         => Ok(Notification::Closed(data)),
            "watcher_joined" => Ok(Notification::WatcherJoined(data)),
            _                => Err(Error::UnknownMessage)
        }
    }

    pub fn send(&self, stream: &mut TcpStream) -> Result<(), Error> {
        let mut encoder = Encoder::new(&mut *stream);

        Ok(try!(self.payload().encode(&mut encoder)))
    }

    fn payload(&self) -> (u8, &str, &String) {
        let header = 2u8;

        match *self {
            Notification::Output(ref string)        => (header, "out",            string),
            Notification::Closed(ref string)        => (header, "closed",         string),
            Notification::WatcherJoined(ref string) => (header, "watcher_joined", string),
        }
    }
}

impl std::error::Error for Error {
    fn description(&self) -> &str {
        "Processing message failed"
    }

    fn cause(&self) -> Option<&::std::error::Error> {
        match *self {
            Error::EncodeError(ref err) => Some(err),
            Error::DecodeError(ref err) => Some(err),
            _ => None
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        std::error::Error::description(self).fmt(f)
    }
}

impl From<msgpack::encode::Error> for Error {
    fn from(err: msgpack::encode::Error) -> Error {
        Error::EncodeError(err)
    }
}

impl From<msgpack::decode::Error> for Error {
    fn from(err: msgpack::decode::Error) -> Error {
        Error::DecodeError(err)
    }
}
<|endoftext|>


<commit_before>extern crate git2;

use git2::*;
use std::process::Command;
use std::path::Path;

pub trait RepoHost  {
    fn remote_url(&self, &str) -> String;
    fn create_project(&self, &str) -> Result<(), Error>;

}

struct Scratch<'a> {
    repo: Repository,
    host: &'a RepoHost,
}

impl<'a> Scratch<'a> {
    pub fn new(path: &Path, host: &'a RepoHost) -> Scratch<'a> {
        Scratch {
            repo: Repository::init_bare(&path).expect("could not init scratch"),
            host: host,
        }
    }

    fn tracking(&self, module: &str, branch: &str) -> Object {
        println!("tracking_branch remotes/{}/{}",module,branch);
        self.repo.revparse_single(&format!("remotes/{}/{}",module,branch))
            .expect("no tracking branch")
    }

    fn call_git(self: &Scratch<'a>, cmd: &str) -> Result<(), Error> {
        println!("Scratch::call_git ({:?}): git {}", &self.repo.path(), &cmd);
        let args: Vec<&str> = cmd.split(" ").collect();
        let repo_path = get_repo_path(&self.repo);
        let mut c = Command::new("git");
        c.env("GIT_DIR", repo_path.as_os_str());
        c.args(&args);

        println!("call_git {:?} command in {:?}", c, repo_path);
        let output = c
            .output()
            .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
        println!("{}{}",
                 String::from_utf8_lossy(&output.stdout),
                 String::from_utf8_lossy(&output.stderr));
        Ok(())
    }

    fn setup(&self,
             central: &str,
             modules: &Vec<String>)
        -> Result<(), Error> {

        println!(" ####### setup scratch repo for remote: {}\n #######\tlocation: {:?}",
            &self.host.remote_url(central),
            &self.repo.path()
        );

        // point remote to central
        if !self.repo.find_remote("central").is_ok() {
            try!(self.repo.remote("central", &self.host.remote_url(central)));
        }

        // create remote for each module
        for module in modules.iter() {
            try!(self.host.create_project(module));

            // create remote for each module
            let remote_name = format!("{}", module);
            let module_remote_url = self.host.remote_url(&module);
            println!("  create remote (remote_name:{}, remote_url:{})",
                &remote_name,
                &module_remote_url
            );
            if !self.repo.find_remote(&remote_name).is_ok() {
                try!(self.repo.remote(&remote_name, &module_remote_url));
            }
        }

        // fetch all branches from remotes
        // FIXME remote branches missing
        // try!(in_tmp_repo(&self.repo, "fetch --all"));
        try!(self.call_git("fetch --all"));
        println!("  fetched all branches from remotes...");
        Ok(())
    }

    // force push of the new revision-object to temp repo
    fn transfer(&self, rev: &str, source: &Path) {
        let target = get_repo_path(&self.repo);
        println!("---> transfer_to_scratch in {}", source.display());
        //" create tmp branch
        call_command("git", &["branch", "-f", "tmp", rev], Some(source));
        // force push
        call_command(
            "git", &["push", "--force", &target.to_string_lossy(), "tmp"], Some(source)
        );
        // delete tmp branch
        call_command("git", &["branch", "-D", "tmp"], Some(source));
        println!("<--- transfer_to_scratch done...");
    }

    // takes everything from base except it's tree and replaces it with the tree given
    fn rewrite(&self, base: &Commit, parents: &[&Commit], tree: &Tree) -> Result<Oid, Error> {
        if parents.len() != 0 {
            try!(self.repo.set_head_detached(parents[0].id()));
        }
        self.repo.commit(Some("HEAD"),
        &base.author(),
        &base.committer(),
        &base.message().unwrap_or("no message"),
        tree,
        parents)
    }

    fn push(&self, oid: Oid, remote: &str) {
        let commit = &self.repo.find_commit(oid).expect("can't find commit");
        self.repo.set_head_detached(commit.id()).expect("can't detach HEAD");
        self.call_git(&format!("push {} HEAD:{}", remote, "master")).expect("can't push");
    }

    fn subtree(&self, tree: &Tree, path: &Path) -> Tree {
        let oid = tree.get_path(path).map(|x| x.id()).expect("can't find subtree");
        return self.repo.find_tree(oid).expect("can't find oid");
    }

    fn module_to_subfolder(&self,
                           module_path: &Path,
                           module_tree: &Tree,
                           master_tree: &Tree)
        -> Result<Oid, Error>
    {
        assert!(module_path.components().count() == 2); // FIXME: drop this requirement
        let parent_path = module_path.parent().expect("module not in subdir");
        let module_name = module_path.file_name().expect("no module name");

        let modules_tree = {
            let mut builder =
                try!(self.repo.treebuilder(Some(&self.subtree(master_tree, parent_path))));
            try!(builder.insert(module_name, module_tree.id(), 0o0040000)); // GIT_FILEMODE_TREE
            try!(builder.write())
        };

        let full_tree = {
            let mut builder = try!(self.repo.treebuilder(Some(master_tree)));
            try!(builder.insert(parent_path, modules_tree, 0o0040000)); // GIT_FILEMODE_TREE
            try!(builder.write())
        };
        println!("module_to_subfolder {}", full_tree);
        Ok(full_tree)
    }
}

pub fn module_review_upload(module: &str,
                            scratch_path: &Path,
                            newrev: &str,
                            central: &str,
                            host: &RepoHost) -> Result<(), Error> {
    println!("in module_review_upload for module {}", &module);
    let scratch = Scratch::new(&scratch_path, host);
    try!(scratch.setup(central,&vec!(module.to_string())));
    try!(scratch.call_git("fetch --all"));

    scratch.transfer(newrev,Path::new("."));
    let mut parent_commit_oid: Oid = scratch.tracking(central, "master").id();

    let object = scratch.tracking(&module, "master").id();
    let oldrev = format!("{}", object);

    {
        let old = try!(scratch.repo.revparse_single(&oldrev).map(|x| x.id()));
        let new = try!(scratch.repo.revparse_single(&newrev).map(|x| x.id()));

        if !try!(scratch.repo.graph_descendant_of(new, old)) {
            println!(".");
            println!("===========================================================");
            println!("======== Commit not based on master, rebase first! ========");
            println!("===========================================================");
            return Ok(());
        }
    }

    let walk = {
        let mut walk = try!(scratch.repo.revwalk());
        walk.set_sorting(SORT_REVERSE | SORT_TIME);
        try!(walk.push_range(&format!("{}..{}", oldrev, newrev)));
        walk
    };

    println!("===== module path: {}", module);
    println!("===== Rewrite commits from {} to {}", oldrev, newrev);

    for rev in walk {
        let newrev = format!("{}", try!(rev));
        if oldrev == newrev {
            continue;
        }
        println!("===== Rewrite commit {}", newrev);

        let module_commit_obj = try!(scratch.repo.revparse_single(&newrev));
        let module_commit = try!(module_commit_obj.as_commit()
                                 .ok_or(Error::from_str("object is not actually a commit")));
        let module_tree = try!(module_commit.tree());

        let parent_commit = try!(scratch.repo.find_commit(parent_commit_oid));

        let new_tree = {
            let master_tree: Tree = try!(parent_commit.tree());
            let new_tree_oid = try!(
                scratch.module_to_subfolder(Path::new(module), &module_tree, &master_tree)
            );
            try!(scratch.repo.find_tree(new_tree_oid))
        };

        parent_commit_oid =
            try!(scratch.rewrite(module_commit, &vec![&parent_commit], &new_tree));
    }

    println!("");
    println!("");
    println!("===================== Doing actual upload in central git ========================");

    let commit = &try!(scratch.repo.find_commit(parent_commit_oid));
    try!(scratch.repo.set_head_detached(commit.id()));
    try!(scratch.call_git(
        &format!("push {} HEAD:{}", &host.remote_url(central), "refs/for/master")
    ));
    println!("==== The review upload may have worked, even if it says error below. Look UP! ====");
    Ok(())
}

pub fn central_submit(newrev: &str,//sha1 of refered commit
                      host: &RepoHost,
                      central: &str,
                      repo_path: &Path,
                      scratch_dir: &Path) -> Result<(), Error> {
    println!(" ---> central_submit (remote addr:{}, sha1 of commit: {})",
        &host.remote_url(central),
        &newrev
    );

    let central_repo = Repository::open(&repo_path)
        .expect(&format!("central repo should exist at {:?}", &repo_path));
    let module_paths = try!(get_module_paths(&central_repo, newrev));

    println!("    ########### SCRATCH: create scratch repo ########### ");
    let scratch = Scratch::new(&scratch_dir,host);
    try!(scratch.setup(
        &central,
        &module_paths,
    ));

    scratch.transfer(newrev, &get_repo_path(&central_repo));
    let central_commit_obj = try!(scratch.repo.revparse_single(newrev));
    let central_commit = try!(central_commit_obj.as_commit()
                              .ok_or(Error::from_str("could not get commit from obj")));
    let central_tree = try!(central_commit.tree());
    println!("    ########### create {} branch in scratch.repo and point to the central_commit ########### ", &central);
    // create central branch in scratch.repo and point to the central_commit
    // is identical to master in central
    // marker for other hooks
    try!(scratch.repo.branch(central, central_commit, true));

    for module_path in module_paths {
        println!(" ####### prepare commit from scratch.repo to module {}", &module_path);
        let module_master_commit_obj = scratch.tracking(&module_path, "master");
        let module_master_commit = try!(module_master_commit_obj.as_commit()
                                        .ok_or(Error::from_str("could not get commit from obj")));

        let parents = vec![module_master_commit];

        println!("\tpath for module in central repository... => {:?}", module_path);

        // new tree is sub-tree of complete central tree
        let new_tree_oid = try!(central_tree.get_path(&Path::new(&module_path))).id();
        let old_tree_oid = try!(module_master_commit.tree()).id();

        // if sha1's are equal the content is equal
        if new_tree_oid != old_tree_oid {
            println!("\tmodule repository for {} is behind, => updating", &module_path);

            let new_tree = try!(scratch.repo.find_tree(new_tree_oid));

            let module_commit =
                try!(scratch.rewrite(central_commit, &parents, &new_tree));
            // do the push to the module git

            scratch.push(module_commit, &host.remote_url(&module_path));
        }
    }
    Ok(())
}

pub fn call_command(command: &str, args: &[&str], cwd: Option<&Path>)
{
    let mut c = Command::new(&command);
    c.args(args);
    println!("call_git {:?} (in {:?})", c, cwd);
    if let Some(path) = cwd {
        // c.env("GIT_DIR", format!("{}.git",path.to_str().unwrap()));
        c.current_dir(path);
    }
    let output = c
        .output()
        .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
    println!("{}{}",
             String::from_utf8_lossy(&output.stdout),
             String::from_utf8_lossy(&output.stderr));
}

fn get_module_paths(central: &Repository, rev: &str) -> Result<Vec<String>, Error> {

    let object = try!(central.revparse_single(rev));
    let commit = try!(object.as_commit()
                      .ok_or(Error::from_str("could not get commit from obj")));
    let tree: Tree = try!(commit.tree());

    let tree_object = try!(tree.get_path(&Path::new("modules")));
    let modules_o = try!(tree_object.to_object(&central));
    let modules = try!(modules_o.as_tree()
                       .ok_or(Error::from_str("could not get tree from path")));

    let mut names = Vec::<String>::new();
    for module in modules.iter() {
        names.push(format!("modules/{}",try!(module.name()
                        .ok_or(Error::from_str("could not get name for module")))
                   .to_string()));
    }
    println!("<--- get_module_paths returns: {:?}", names);
    Ok(names)
}

pub fn get_repo_path(repo: &Repository) -> &Path {
    if repo.is_bare() {
        return repo.path();
    }
    return repo.workdir().expect("get workdir from repo");
}

<commit_msg>more reliable remote tracking<commit_after>extern crate git2;

use git2::*;
use std::process::Command;
use std::path::Path;

pub trait RepoHost  {
    fn remote_url(&self, &str) -> String;
    fn create_project(&self, &str) -> Result<(), Error>;

}

struct Scratch<'a> {
    repo: Repository,
    host: &'a RepoHost,
}

impl<'a> Scratch<'a> {
    pub fn new(path: &Path, host: &'a RepoHost) -> Scratch<'a> {
        Scratch {
            repo: Repository::init_bare(&path).expect("could not init scratch"),
            host: host,
        }
    }

    fn tracking(&self, module: &str, branch: &str) -> Object {
        println!("tracking_branch remotes/{}/{}",module,branch);

        let remote_name = format!("{}", module);
        let remote_url = self.host.remote_url(&module);
        println!("  create remote (remote_name:{}, remote_url:{})",
            &remote_name,
            &remote_url
        );
        if !self.repo.find_remote(&remote_name).is_ok() {
            self.repo.remote(&remote_name, &remote_url).expect("can't create remote");
        }

        self.call_git("fetch --all").expect("could not fetch");
        self.repo.revparse_single(&format!("remotes/{}/{}",module,branch))
            .expect("no tracking branch")
    }

    fn call_git(self: &Scratch<'a>, cmd: &str) -> Result<(), Error> {
        println!("Scratch::call_git ({:?}): git {}", &self.repo.path(), &cmd);
        let args: Vec<&str> = cmd.split(" ").collect();
        let repo_path = get_repo_path(&self.repo);
        let mut c = Command::new("git");
        c.env("GIT_DIR", repo_path.as_os_str());
        c.args(&args);

        println!("call_git {:?} command in {:?}", c, repo_path);
        let output = c
            .output()
            .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
        println!("{}{}",
                 String::from_utf8_lossy(&output.stdout),
                 String::from_utf8_lossy(&output.stderr));
        Ok(())
    }

    fn setup(&self,
             central: &str,
             rev: &str)
        -> Result<(), Error> {

        println!(" ####### setup scratch repo for remote: {}\n #######\tlocation: {:?}",
            &self.host.remote_url(central),
            &self.repo.path()
        );

        // create remote for each module
        for module in self.module_paths(rev).unwrap().iter() {
            try!(self.host.create_project(module));

        }

        Ok(())
    }

    // force push of the new revision-object to temp repo
    fn transfer(&self, rev: &str, source: &Path) {
        let target = get_repo_path(&self.repo);
        println!("---> transfer_to_scratch in {}", source.display());
        //" create tmp branch
        call_command("git", &["branch", "-f", "tmp", rev], Some(source));
        // force push
        call_command(
            "git", &["push", "--force", &target.to_string_lossy(), "tmp"], Some(source)
        );
        // delete tmp branch
        call_command("git", &["branch", "-D", "tmp"], Some(source));
        println!("<--- transfer_to_scratch done...");
    }

    // takes everything from base except it's tree and replaces it with the tree given
    fn rewrite(&self, base: &Commit, parents: &[&Commit], tree: &Tree) -> Result<Oid, Error> {
        if parents.len() != 0 {
            try!(self.repo.set_head_detached(parents[0].id()));
        }
        self.repo.commit(Some("HEAD"),
        &base.author(),
        &base.committer(),
        &base.message().unwrap_or("no message"),
        tree,
        parents)
    }

    fn push(&self, oid: Oid, remote: &str) {
        let commit = &self.repo.find_commit(oid).expect("can't find commit");
        self.repo.set_head_detached(commit.id()).expect("can't detach HEAD");
        self.call_git(&format!("push {} HEAD:{}", remote, "master")).expect("can't push");
    }

    fn subtree(&self, tree: &Tree, path: &Path) -> Tree {
        let oid = tree.get_path(path).map(|x| x.id()).expect("can't find subtree");
        return self.repo.find_tree(oid).expect("can't find oid");
    }

    fn module_to_subfolder(&self,
                           module_path: &Path,
                           module_tree: &Tree,
                           master_tree: &Tree)
        -> Result<Oid, Error>
    {
        assert!(module_path.components().count() == 2); // FIXME: drop this requirement
        let parent_path = module_path.parent().expect("module not in subdir");
        let module_name = module_path.file_name().expect("no module name");

        let modules_tree = {
            let mut builder =
                try!(self.repo.treebuilder(Some(&self.subtree(master_tree, parent_path))));
            try!(builder.insert(module_name, module_tree.id(), 0o0040000)); // GIT_FILEMODE_TREE
            try!(builder.write())
        };

        let full_tree = {
            let mut builder = try!(self.repo.treebuilder(Some(master_tree)));
            try!(builder.insert(parent_path, modules_tree, 0o0040000)); // GIT_FILEMODE_TREE
            try!(builder.write())
        };
        println!("module_to_subfolder {}", full_tree);
        Ok(full_tree)
    }

    fn module_paths(&self, rev: &str) -> Result<Vec<String>, Error> {
        println!("---> get_module_paths");
        let object = try!(self.repo.revparse_single(rev));
        let commit = try!(object.as_commit()
                          .ok_or(Error::from_str("could not get commit from obj")));
        let tree: Tree = try!(commit.tree());

        let tree_object = try!(tree.get_path(&Path::new("modules")));
        let modules_o = try!(tree_object.to_object(&self.repo));
        let modules = try!(modules_o.as_tree()
                           .ok_or(Error::from_str("could not get tree from path")));

        let mut names = Vec::<String>::new();
        for module in modules.iter() {
            names.push(format!("modules/{}",try!(module.name()
                            .ok_or(Error::from_str("could not get name for module")))
                       .to_string()));
        }
        println!("<--- get_module_paths returns: {:?}", names);
        Ok(names)
    }
}

pub fn module_review_upload(module: &str,
                            scratch_path: &Path,
                            newrev: &str,
                            central: &str,
                            host: &RepoHost) -> Result<(), Error> {
    println!("in module_review_upload for module {}", &module);
    let scratch = Scratch::new(&scratch_path, host);
    scratch.transfer(newrev,Path::new("."));
    try!(scratch.setup(central,&format!("{}",scratch.tracking(central,"master").id())));

    let mut parent_commit_oid: Oid = scratch.tracking(central, "master").id();

    let object = scratch.tracking(&module, "master").id();
    let oldrev = format!("{}", object);

    {
        let old = try!(scratch.repo.revparse_single(&oldrev).map(|x| x.id()));
        let new = try!(scratch.repo.revparse_single(&newrev).map(|x| x.id()));

        if !try!(scratch.repo.graph_descendant_of(new, old)) {
            println!(".");
            println!("===========================================================");
            println!("======== Commit not based on master, rebase first! ========");
            println!("===========================================================");
            return Ok(());
        }
    }

    let walk = {
        let mut walk = try!(scratch.repo.revwalk());
        walk.set_sorting(SORT_REVERSE | SORT_TIME);
        try!(walk.push_range(&format!("{}..{}", oldrev, newrev)));
        walk
    };

    println!("===== module path: {}", module);
    println!("===== Rewrite commits from {} to {}", oldrev, newrev);

    for rev in walk {
        let newrev = format!("{}", try!(rev));
        if oldrev == newrev {
            continue;
        }
        println!("===== Rewrite commit {}", newrev);

        let module_commit_obj = try!(scratch.repo.revparse_single(&newrev));
        let module_commit = try!(module_commit_obj.as_commit()
                                 .ok_or(Error::from_str("object is not actually a commit")));
        let module_tree = try!(module_commit.tree());

        let parent_commit = try!(scratch.repo.find_commit(parent_commit_oid));

        let new_tree = {
            let master_tree: Tree = try!(parent_commit.tree());
            let new_tree_oid = try!(
                scratch.module_to_subfolder(Path::new(module), &module_tree, &master_tree)
            );
            try!(scratch.repo.find_tree(new_tree_oid))
        };

        parent_commit_oid =
            try!(scratch.rewrite(module_commit, &vec![&parent_commit], &new_tree));
    }

    println!("");
    println!("");
    println!("===================== Doing actual upload in central git ========================");

    let commit = &try!(scratch.repo.find_commit(parent_commit_oid));
    try!(scratch.repo.set_head_detached(commit.id()));
    try!(scratch.call_git(
        &format!("push {} HEAD:{}", &host.remote_url(central), "refs/for/master")
    ));
    println!("==== The review upload may have worked, even if it says error below. Look UP! ====");
    Ok(())
}

pub fn central_submit(newrev: &str,//sha1 of refered commit
                      host: &RepoHost,
                      central: &str,
                      repo_path: &Path,
                      scratch_dir: &Path) -> Result<(), Error> {
    println!(" ---> central_submit (remote addr:{}, sha1 of commit: {})",
        &host.remote_url(central),
        &newrev
    );

    println!("    ########### SCRATCH: create scratch repo ########### ");
    let scratch = Scratch::new(&scratch_dir,host);
    scratch.transfer(newrev, &repo_path);
    try!(scratch.setup(
        &central,
        newrev,
    ));

    let central_commit_obj = try!(scratch.repo.revparse_single(newrev));
    let central_commit = try!(central_commit_obj.as_commit()
                              .ok_or(Error::from_str("could not get commit from obj")));
    let central_tree = try!(central_commit.tree());
    println!("    ########### create {} branch in scratch.repo and point to the central_commit ########### ", &central);
    // create central branch in scratch.repo and point to the central_commit
    // is identical to master in central
    // marker for other hooks
    try!(scratch.repo.branch(central, central_commit, true));

    for module_path in scratch.module_paths(newrev).unwrap() {
        println!(" ####### prepare commit from scratch.repo to module {}", &module_path);
        let module_master_commit_obj = scratch.tracking(&module_path, "master");
        let module_master_commit = try!(module_master_commit_obj.as_commit()
                                        .ok_or(Error::from_str("could not get commit from obj")));

        let parents = vec![module_master_commit];

        println!("\tpath for module in central repository... => {:?}", module_path);

        // new tree is sub-tree of complete central tree
        let new_tree_oid = try!(central_tree.get_path(&Path::new(&module_path))).id();
        let old_tree_oid = try!(module_master_commit.tree()).id();

        // if sha1's are equal the content is equal
        if new_tree_oid != old_tree_oid {
            println!("\tmodule repository for {} is behind, => updating", &module_path);

            let new_tree = try!(scratch.repo.find_tree(new_tree_oid));

            let module_commit =
                try!(scratch.rewrite(central_commit, &parents, &new_tree));
            // do the push to the module git

            scratch.push(module_commit, &host.remote_url(&module_path));
        }
    }
    Ok(())
}

pub fn call_command(command: &str, args: &[&str], cwd: Option<&Path>)
{
    let mut c = Command::new(&command);
    c.args(args);
    println!("call_git {:?} (in {:?})", c, cwd);
    if let Some(path) = cwd {
        // c.env("GIT_DIR", format!("{}.git",path.to_str().unwrap()));
        c.current_dir(path);
    }
    let output = c
        .output()
        .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
    println!("{}{}",
             String::from_utf8_lossy(&output.stdout),
             String::from_utf8_lossy(&output.stderr));
}

pub fn get_repo_path(repo: &Repository) -> &Path {
    if repo.is_bare() {
        return repo.path();
    }
    return repo.workdir().expect("get workdir from repo");
}

<|endoftext|>