<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

//! Loader for a single run, with one or more event files.

use log::warn;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use std::sync::RwLock;

use crate::commit;
use crate::data_compat::{EventValue, SummaryValue};
use crate::event_file::EventFileReader;
use crate::proto::tensorboard as pb;
use crate::reservoir::StageReservoir;
use crate::types::{Step, Tag, WallTime};

/// A loader to accumulate reservoir-sampled events in a single TensorBoard run.
///
/// For now, a run loader always reads from [`File`]s on disk. In the future, this may be
/// parameterized over a filesystem interface.
#[derive(Debug)]
pub struct RunLoader {
    /// The earliest event `wall_time` seen in any event file in this run.
    ///
    /// This is `None` if and only if no events have been seen. Its value may decrease as new
    /// events are read, but in practice this is expected to be the wall time of the first
    /// `file_version` event in the first event file.
    start_time: Option<WallTime>,

    /// The event files in this run.
    ///
    /// Event files are sorted and read lexicographically by name, which is designed to coincide
    /// with actual start time. See [`EventFile::Dead`] for conditions under which an event file
    /// may be dead. Once an event file is added to this map, it may become dead, but it will not
    /// be removed entirely. This way, we know not to just re-open it again at the next load cycle.
    files: BTreeMap<PathBuf, EventFile<BufReader<File>>>,

    /// Reservoir-sampled data and metadata for each time series.
    time_series: HashMap<Tag, StageTimeSeries>,
}

#[derive(Debug)]
enum EventFile<R> {
    /// An event file that may still have more valid data.
    Active(EventFileReader<R>),
    /// An event file that can no longer be read.
    ///
    /// This can be because of a non-recoverable read error (e.g., a bad length checksum), due to
    /// the last-read record being very old (note: not yet implemented), or due to the file being
    /// deleted.
    Dead,
}

#[derive(Debug)]
struct StageTimeSeries {
    data_class: pb::DataClass,
    metadata: Box<pb::SummaryMetadata>,
    rsv: StageReservoir<StageValue>,
}

/// A value staged in the reservoir.
///
/// This is kept as close as possible to the on-disk event representation, since every record in
/// the stream is converted into this format.
#[derive(Debug)]
struct StageValue {
    wall_time: WallTime,
    payload: EventValue,
}

impl StageTimeSeries {
    fn new(metadata: Box<pb::SummaryMetadata>) -> Self {
        let data_class =
            pb::DataClass::from_i32(metadata.data_class).unwrap_or(pb::DataClass::Unknown);
        let capacity = match data_class {
            pb::DataClass::Scalar => 1000,
            pb::DataClass::Tensor => 100,
            pb::DataClass::BlobSequence => 10,
            _ => 0,
        };
        Self {
            data_class,
            metadata,
            rsv: StageReservoir::new(capacity),
        }
    }

    /// Writes all staged data for this time series into the commit.
    fn commit(&mut self, tag: &Tag, run: &mut commit::RunData) {
        use pb::DataClass;
        match self.data_class {
            DataClass::Scalar => self.commit_to(tag, &mut run.scalars, EventValue::into_scalar),
            DataClass::Tensor => todo!("tensor time series not yet supported"),
            DataClass::BlobSequence => todo!("blob sequence time series not yet supported"),
            _ => (),
        };
    }

    /// Helper for `commit`: writes staged data for this time series into storage for a statically
    /// known data class.
    fn commit_to<V, F: FnMut(EventValue) -> Result<V, commit::DataLoss>>(
        &mut self,
        tag: &Tag,
        store: &mut commit::TagStore<V>,
        mut enrich: F,
    ) {
        let commit_ts = store
            .entry(tag.clone())
            .or_insert_with(|| commit::TimeSeries::new(self.metadata.clone()));
        self.rsv
            .commit_map(&mut commit_ts.basin, |StageValue { wall_time, payload }| {
                (wall_time, enrich(payload))
            });
    }
}

impl RunLoader {
    pub fn new() -> Self {
        Self {
            start_time: None,
            files: BTreeMap::new(),
            time_series: HashMap::new(),
        }
    }

    /// Loads new data given the current set of event files.
    ///
    /// The provided filenames should correspond to the entire set of event files currently part of
    /// this run.
    ///
    /// The given commit must have an entry for this run (the entry may be empty).
    ///
    /// # Panics
    ///
    /// If we need to access `run_data` but the lock is poisoned.
    pub fn reload(&mut self, filenames: Vec<PathBuf>, run_data: &RwLock<commit::RunData>) {
        self.update_file_set(filenames);
        self.reload_files();
        self.commit_all(run_data);
    }

    /// Updates the active key set of `self.files` to match the given filenames.
    ///
    /// After this function returns, `self.files` may still have keys not in `filenames`, but they
    /// will all map to [`EventFile::Dead`].
    fn update_file_set(&mut self, filenames: Vec<PathBuf>) {
        // Remove any discarded files.
        let new_file_set: HashSet<&PathBuf> = filenames.iter().collect();
        for (k, v) in self.files.iter_mut() {
            if !new_file_set.contains(k) {
                *v = EventFile::Dead;
            }
        }

        // Open readers for any new files.
        for filename in filenames {
            use std::collections::btree_map::Entry;
            match self.files.entry(filename) {
                Entry::Occupied(_) => {}
                Entry::Vacant(v) => {
                    let event_file = match File::open(v.key()) {
                        Ok(file) => {
                            let reader = EventFileReader::new(BufReader::new(file));
                            EventFile::Active(reader)
                        }
                        // TODO(@wchargin): Improve error handling?
                        Err(e) => {
                            warn!("Failed to open event file {:?}: {:?}", v.key(), e);
                            EventFile::Dead
                        }
                    };
                    v.insert(event_file);
                }
            };
        }
    }

    /// Reads data from all active event files.
    fn reload_files(&mut self) {
        for (filename, ef) in self.files.iter_mut() {
            let reader = match ef {
                EventFile::Dead => continue,
                EventFile::Active(reader) => reader,
            };

            loop {
                use crate::event_file::ReadEventError::ReadRecordError;
                use crate::tf_record::ReadRecordError::Truncated;
                let event = match reader.read_event() {
                    Ok(event) => event,
                    Err(ReadRecordError(Truncated)) => break,
                    Err(e) => {
                        // TODO(@wchargin): Improve error handling?
                        warn!("Read error in {}: {:?}", filename.display(), e);
                        *ef = EventFile::Dead;
                        break;
                    }
                };
                read_event(&mut self.time_series, &mut self.start_time, event);
            }
        }
    }

    fn commit_all(&mut self, run_data: &RwLock<commit::RunData>) {
        let mut run = run_data.write().expect("acquiring tags lock");
        run.start_time = self.start_time;
        for (tag, ts) in &mut self.time_series {
            ts.commit(tag, &mut *run);
        }
    }
}

/// Reads a single event into the structures of a run loader.
///
/// This is a standalone function because it's called from `reload_files` in a context that already
/// has an exclusive reference into `self.files`, and so can't call methods on `&mut self`.
fn read_event(
    time_series: &mut HashMap<Tag, StageTimeSeries>,
    start_time: &mut Option<WallTime>,
    e: pb::Event,
) {
    let step = Step(e.step);
    let wall_time = match WallTime::new(e.wall_time) {
        None => {
            // TODO(@wchargin): Improve error handling.
            warn!(
                "Dropping event at step {} with invalid wall time {}",
                e.step, e.wall_time
            );
            return;
        }
        Some(wt) => wt,
    };
    if start_time.map(|start| wall_time < start).unwrap_or(true) {
        *start_time = Some(wall_time);
    }
    match e.what {
        Some(pb::event::What::GraphDef(_)) => {
            // TODO(@wchargin): Handle run graphs.
            warn!("`graph_def` events not yet handled");
        }
        Some(pb::event::What::Summary(sum)) => {
            for mut summary_pb_value in sum.value {
                let summary_value = match summary_pb_value.value {
                    None => continue,
                    Some(v) => SummaryValue(Box::new(v)),
                };

                use std::collections::hash_map::Entry;
                let ts = match time_series.entry(Tag(summary_pb_value.tag)) {
                    Entry::Occupied(o) => o.into_mut(),
                    Entry::Vacant(v) => {
                        let metadata =
                            summary_value.initial_metadata(summary_pb_value.metadata.take());
                        v.insert(StageTimeSeries::new(metadata))
                    }
                };
                let sv = StageValue {
                    wall_time,
                    payload: EventValue::Summary(summary_value),
                };
                ts.rsv.offer(step, sv);
            }
        }
        _ => {}
    }
}

impl Default for RunLoader {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::fs::File;
    use std::io::BufWriter;

    use crate::commit::Commit;
    use crate::types::Run;
    use crate::writer::SummaryWriteExt;

    #[test]
    fn test() -> Result<(), Box<dyn std::error::Error>> {
        let logdir = tempfile::tempdir()?;
        let f1_name = logdir.path().join("tfevents.123");
        let f2_name = logdir.path().join("tfevents.456");
        let mut f1 = BufWriter::new(File::create(&f1_name)?);
        let mut f2 = BufWriter::new(File::create(&f2_name)?);

        // Write file versions.
        for (f, wall_time) in &mut [(&mut f1, 1234.0), (&mut f2, 2345.0)] {
            let file_version = pb::Event {
                wall_time: *wall_time,
                what: Some(pb::event::What::FileVersion("brain.Event:2".to_string())),
                ..Default::default()
            };
            f.write_event(&file_version)?;
        }

        // Write some data points across both files.
        let run = Run("train".to_string());
        let tag = Tag("accuracy".to_string());
        f1.write_scalar(&tag, Step(0), WallTime::new(1235.0).unwrap(), 0.25)?;
        f1.write_scalar(&tag, Step(1), WallTime::new(1236.0).unwrap(), 0.50)?;
        f1.write_scalar(&tag, Step(2), WallTime::new(1237.0).unwrap(), 0.75)?;
        f1.write_scalar(&tag, Step(3), WallTime::new(1238.0).unwrap(), 1.00)?;
        // preempt!
        f2.write_scalar(&tag, Step(2), WallTime::new(2346.0).unwrap(), 0.70)?;
        f2.write_scalar(&tag, Step(3), WallTime::new(2347.0).unwrap(), 0.85)?;
        f2.write_scalar(&tag, Step(4), WallTime::new(2348.0).unwrap(), 0.90)?;
        // flush, so that the data's there when we read it
        f1.into_inner()?.sync_all()?;
        f2.into_inner()?.sync_all()?;

        let mut loader = RunLoader::new();
        let commit = Commit::new();
        commit
            .runs
            .write()
            .expect("write-locking runs map")
            .insert(run.clone(), Default::default());
        loader.reload(vec![f1_name, f2_name], &commit.runs.read().unwrap()[&run]);

        // Start time should be that of the file version event, even though that didn't correspond
        // to any time series.
        assert_eq!(loader.start_time, Some(WallTime::new(1234.0).unwrap()));

        let runs = commit.runs.read().expect("read-locking runs map");
        let run_data: &commit::RunData = &*runs
            .get(&run)
            .expect("looking up data for run")
            .read()
            .expect("read-locking run data map");

        assert_eq!(run_data.scalars.keys().collect::<Vec<_>>(), vec![&tag]);
        let ts = run_data.scalars.get(&tag).unwrap();
        assert_eq!(
            *ts.metadata,
            pb::SummaryMetadata {
                plugin_data: Some(pb::summary_metadata::PluginData {
                    plugin_name: crate::data_compat::SCALARS_PLUGIN_NAME.to_string(),
                    ..Default::default()
                }),
                data_class: pb::DataClass::Scalar.into(),
                ..Default::default()
            }
        );

        // Points should be as expected (no downsampling at these sizes).
        let scalar = commit::ScalarValue;
        assert_eq!(
            ts.valid_values().collect::<Vec<_>>(),
            vec![
                (Step(0), WallTime::new(1235.0).unwrap(), &scalar(0.25)),
                (Step(1), WallTime::new(1236.0).unwrap(), &scalar(0.50)),
                (Step(2), WallTime::new(2346.0).unwrap(), &scalar(0.70)),
                (Step(3), WallTime::new(2347.0).unwrap(), &scalar(0.85)),
                (Step(4), WallTime::new(2348.0).unwrap(), &scalar(0.90)),
            ]
        );

        Ok(())
    }
}
<commit_msg>rust: fix panic when reading explicit tensors/blobs (#4441)<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

//! Loader for a single run, with one or more event files.

use log::warn;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use std::sync::RwLock;

use crate::commit;
use crate::data_compat::{EventValue, SummaryValue};
use crate::event_file::EventFileReader;
use crate::proto::tensorboard as pb;
use crate::reservoir::StageReservoir;
use crate::types::{Step, Tag, WallTime};

/// A loader to accumulate reservoir-sampled events in a single TensorBoard run.
///
/// For now, a run loader always reads from [`File`]s on disk. In the future, this may be
/// parameterized over a filesystem interface.
#[derive(Debug)]
pub struct RunLoader {
    /// The earliest event `wall_time` seen in any event file in this run.
    ///
    /// This is `None` if and only if no events have been seen. Its value may decrease as new
    /// events are read, but in practice this is expected to be the wall time of the first
    /// `file_version` event in the first event file.
    start_time: Option<WallTime>,

    /// The event files in this run.
    ///
    /// Event files are sorted and read lexicographically by name, which is designed to coincide
    /// with actual start time. See [`EventFile::Dead`] for conditions under which an event file
    /// may be dead. Once an event file is added to this map, it may become dead, but it will not
    /// be removed entirely. This way, we know not to just re-open it again at the next load cycle.
    files: BTreeMap<PathBuf, EventFile<BufReader<File>>>,

    /// Reservoir-sampled data and metadata for each time series.
    time_series: HashMap<Tag, StageTimeSeries>,
}

#[derive(Debug)]
enum EventFile<R> {
    /// An event file that may still have more valid data.
    Active(EventFileReader<R>),
    /// An event file that can no longer be read.
    ///
    /// This can be because of a non-recoverable read error (e.g., a bad length checksum), due to
    /// the last-read record being very old (note: not yet implemented), or due to the file being
    /// deleted.
    Dead,
}

#[derive(Debug)]
struct StageTimeSeries {
    data_class: pb::DataClass,
    metadata: Box<pb::SummaryMetadata>,
    rsv: StageReservoir<StageValue>,
}

/// A value staged in the reservoir.
///
/// This is kept as close as possible to the on-disk event representation, since every record in
/// the stream is converted into this format.
#[derive(Debug)]
struct StageValue {
    wall_time: WallTime,
    payload: EventValue,
}

impl StageTimeSeries {
    fn new(metadata: Box<pb::SummaryMetadata>) -> Self {
        let data_class =
            pb::DataClass::from_i32(metadata.data_class).unwrap_or(pb::DataClass::Unknown);
        let capacity = match data_class {
            pb::DataClass::Scalar => 1000,
            pb::DataClass::Tensor => 100,
            pb::DataClass::BlobSequence => 10,
            _ => 0,
        };
        Self {
            data_class,
            metadata,
            rsv: StageReservoir::new(capacity),
        }
    }

    /// Writes all staged data for this time series into the commit.
    fn commit(&mut self, tag: &Tag, run: &mut commit::RunData) {
        use pb::DataClass;
        match self.data_class {
            DataClass::Scalar => self.commit_to(tag, &mut run.scalars, EventValue::into_scalar),
            DataClass::Tensor => {
                warn!(
                    "Tensor time series not yet supported (tag: {:?}, plugin: {:?})",
                    tag.0,
                    self.metadata
                        .plugin_data
                        .as_ref()
                        .map(|p| p.plugin_name.as_str())
                        .unwrap_or("")
                );
            }
            DataClass::BlobSequence => {
                warn!(
                    "Blob sequence time series not yet supported (tag: {}, plugin: {})",
                    tag.0,
                    self.metadata
                        .plugin_data
                        .as_ref()
                        .map(|p| p.plugin_name.as_str())
                        .unwrap_or("")
                );
            }
            _ => (),
        };
    }

    /// Helper for `commit`: writes staged data for this time series into storage for a statically
    /// known data class.
    fn commit_to<V, F: FnMut(EventValue) -> Result<V, commit::DataLoss>>(
        &mut self,
        tag: &Tag,
        store: &mut commit::TagStore<V>,
        mut enrich: F,
    ) {
        let commit_ts = store
            .entry(tag.clone())
            .or_insert_with(|| commit::TimeSeries::new(self.metadata.clone()));
        self.rsv
            .commit_map(&mut commit_ts.basin, |StageValue { wall_time, payload }| {
                (wall_time, enrich(payload))
            });
    }
}

impl RunLoader {
    pub fn new() -> Self {
        Self {
            start_time: None,
            files: BTreeMap::new(),
            time_series: HashMap::new(),
        }
    }

    /// Loads new data given the current set of event files.
    ///
    /// The provided filenames should correspond to the entire set of event files currently part of
    /// this run.
    ///
    /// The given commit must have an entry for this run (the entry may be empty).
    ///
    /// # Panics
    ///
    /// If we need to access `run_data` but the lock is poisoned.
    pub fn reload(&mut self, filenames: Vec<PathBuf>, run_data: &RwLock<commit::RunData>) {
        self.update_file_set(filenames);
        self.reload_files();
        self.commit_all(run_data);
    }

    /// Updates the active key set of `self.files` to match the given filenames.
    ///
    /// After this function returns, `self.files` may still have keys not in `filenames`, but they
    /// will all map to [`EventFile::Dead`].
    fn update_file_set(&mut self, filenames: Vec<PathBuf>) {
        // Remove any discarded files.
        let new_file_set: HashSet<&PathBuf> = filenames.iter().collect();
        for (k, v) in self.files.iter_mut() {
            if !new_file_set.contains(k) {
                *v = EventFile::Dead;
            }
        }

        // Open readers for any new files.
        for filename in filenames {
            use std::collections::btree_map::Entry;
            match self.files.entry(filename) {
                Entry::Occupied(_) => {}
                Entry::Vacant(v) => {
                    let event_file = match File::open(v.key()) {
                        Ok(file) => {
                            let reader = EventFileReader::new(BufReader::new(file));
                            EventFile::Active(reader)
                        }
                        // TODO(@wchargin): Improve error handling?
                        Err(e) => {
                            warn!("Failed to open event file {:?}: {:?}", v.key(), e);
                            EventFile::Dead
                        }
                    };
                    v.insert(event_file);
                }
            };
        }
    }

    /// Reads data from all active event files.
    fn reload_files(&mut self) {
        for (filename, ef) in self.files.iter_mut() {
            let reader = match ef {
                EventFile::Dead => continue,
                EventFile::Active(reader) => reader,
            };

            loop {
                use crate::event_file::ReadEventError::ReadRecordError;
                use crate::tf_record::ReadRecordError::Truncated;
                let event = match reader.read_event() {
                    Ok(event) => event,
                    Err(ReadRecordError(Truncated)) => break,
                    Err(e) => {
                        // TODO(@wchargin): Improve error handling?
                        warn!("Read error in {}: {:?}", filename.display(), e);
                        *ef = EventFile::Dead;
                        break;
                    }
                };
                read_event(&mut self.time_series, &mut self.start_time, event);
            }
        }
    }

    fn commit_all(&mut self, run_data: &RwLock<commit::RunData>) {
        let mut run = run_data.write().expect("acquiring tags lock");
        run.start_time = self.start_time;
        for (tag, ts) in &mut self.time_series {
            ts.commit(tag, &mut *run);
        }
    }
}

/// Reads a single event into the structures of a run loader.
///
/// This is a standalone function because it's called from `reload_files` in a context that already
/// has an exclusive reference into `self.files`, and so can't call methods on `&mut self`.
fn read_event(
    time_series: &mut HashMap<Tag, StageTimeSeries>,
    start_time: &mut Option<WallTime>,
    e: pb::Event,
) {
    let step = Step(e.step);
    let wall_time = match WallTime::new(e.wall_time) {
        None => {
            // TODO(@wchargin): Improve error handling.
            warn!(
                "Dropping event at step {} with invalid wall time {}",
                e.step, e.wall_time
            );
            return;
        }
        Some(wt) => wt,
    };
    if start_time.map(|start| wall_time < start).unwrap_or(true) {
        *start_time = Some(wall_time);
    }
    match e.what {
        Some(pb::event::What::GraphDef(_)) => {
            // TODO(@wchargin): Handle run graphs.
            warn!("`graph_def` events not yet handled");
        }
        Some(pb::event::What::Summary(sum)) => {
            for mut summary_pb_value in sum.value {
                let summary_value = match summary_pb_value.value {
                    None => continue,
                    Some(v) => SummaryValue(Box::new(v)),
                };

                use std::collections::hash_map::Entry;
                let ts = match time_series.entry(Tag(summary_pb_value.tag)) {
                    Entry::Occupied(o) => o.into_mut(),
                    Entry::Vacant(v) => {
                        let metadata =
                            summary_value.initial_metadata(summary_pb_value.metadata.take());
                        v.insert(StageTimeSeries::new(metadata))
                    }
                };
                let sv = StageValue {
                    wall_time,
                    payload: EventValue::Summary(summary_value),
                };
                ts.rsv.offer(step, sv);
            }
        }
        _ => {}
    }
}

impl Default for RunLoader {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::fs::File;
    use std::io::BufWriter;

    use crate::commit::Commit;
    use crate::types::Run;
    use crate::writer::SummaryWriteExt;

    #[test]
    fn test() -> Result<(), Box<dyn std::error::Error>> {
        let logdir = tempfile::tempdir()?;
        let f1_name = logdir.path().join("tfevents.123");
        let f2_name = logdir.path().join("tfevents.456");
        let mut f1 = BufWriter::new(File::create(&f1_name)?);
        let mut f2 = BufWriter::new(File::create(&f2_name)?);

        // Write file versions.
        for (f, wall_time) in &mut [(&mut f1, 1234.0), (&mut f2, 2345.0)] {
            let file_version = pb::Event {
                wall_time: *wall_time,
                what: Some(pb::event::What::FileVersion("brain.Event:2".to_string())),
                ..Default::default()
            };
            f.write_event(&file_version)?;
        }

        // Write some data points across both files.
        let run = Run("train".to_string());
        let tag = Tag("accuracy".to_string());
        f1.write_scalar(&tag, Step(0), WallTime::new(1235.0).unwrap(), 0.25)?;
        f1.write_scalar(&tag, Step(1), WallTime::new(1236.0).unwrap(), 0.50)?;
        f1.write_scalar(&tag, Step(2), WallTime::new(1237.0).unwrap(), 0.75)?;
        f1.write_scalar(&tag, Step(3), WallTime::new(1238.0).unwrap(), 1.00)?;
        // preempt!
        f2.write_scalar(&tag, Step(2), WallTime::new(2346.0).unwrap(), 0.70)?;
        f2.write_scalar(&tag, Step(3), WallTime::new(2347.0).unwrap(), 0.85)?;
        f2.write_scalar(&tag, Step(4), WallTime::new(2348.0).unwrap(), 0.90)?;
        // flush, so that the data's there when we read it
        f1.into_inner()?.sync_all()?;
        f2.into_inner()?.sync_all()?;

        let mut loader = RunLoader::new();
        let commit = Commit::new();
        commit
            .runs
            .write()
            .expect("write-locking runs map")
            .insert(run.clone(), Default::default());
        loader.reload(vec![f1_name, f2_name], &commit.runs.read().unwrap()[&run]);

        // Start time should be that of the file version event, even though that didn't correspond
        // to any time series.
        assert_eq!(loader.start_time, Some(WallTime::new(1234.0).unwrap()));

        let runs = commit.runs.read().expect("read-locking runs map");
        let run_data: &commit::RunData = &*runs
            .get(&run)
            .expect("looking up data for run")
            .read()
            .expect("read-locking run data map");

        assert_eq!(run_data.scalars.keys().collect::<Vec<_>>(), vec![&tag]);
        let ts = run_data.scalars.get(&tag).unwrap();
        assert_eq!(
            *ts.metadata,
            pb::SummaryMetadata {
                plugin_data: Some(pb::summary_metadata::PluginData {
                    plugin_name: crate::data_compat::SCALARS_PLUGIN_NAME.to_string(),
                    ..Default::default()
                }),
                data_class: pb::DataClass::Scalar.into(),
                ..Default::default()
            }
        );

        // Points should be as expected (no downsampling at these sizes).
        let scalar = commit::ScalarValue;
        assert_eq!(
            ts.valid_values().collect::<Vec<_>>(),
            vec![
                (Step(0), WallTime::new(1235.0).unwrap(), &scalar(0.25)),
                (Step(1), WallTime::new(1236.0).unwrap(), &scalar(0.50)),
                (Step(2), WallTime::new(2346.0).unwrap(), &scalar(0.70)),
                (Step(3), WallTime::new(2347.0).unwrap(), &scalar(0.85)),
                (Step(4), WallTime::new(2348.0).unwrap(), &scalar(0.90)),
            ]
        );

        Ok(())
    }
}
<|endoftext|>


<commit_before>use uuid::{Builder, Uuid, Variant, Version};

use crate::{Dummy, Faker};
use crate::faker::uuid::raw::*;
use crate::locales::Data;

impl<L: Data> Dummy<UuidV1<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV1<L>, rng: &mut R) -> Self {
      let bytes: [u8;8] = rng.gen();
      let (ticks, _) = config.1.to_rfc4122();
      let time_low = (ticks & 0xFFFF_FFFF) as u32;
      let time_mid = ((ticks >> 32) & 0xFFFF) as u16;
      let time_high_and_version = (((ticks >> 48) & 0x0FFF) as u16) | (1 << 12);
      Uuid::from_fields(time_low, time_mid, time_high_and_version, &bytes).expect("generate uuid::Uuid::from_fields")
    }

    fn dummy(UuidV1(_, timestamp, node_id): &UuidV1<L>) -> Self {
        Self::new_v1(*timestamp, node_id).expect("generate uuid::Uuid::new_v1")
    }
}

impl<L: Data> Dummy<UuidV1<L>> for String {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV1<L>, rng: &mut R) -> Self {
        Uuid::dummy_with_rng(config, rng).to_hyphenated().to_string()
    }

    fn dummy(config: &UuidV1<L>) -> Self {
        Uuid::dummy(config).to_hyphenated().to_string()
    }
}

impl<L: Data> Dummy<UuidV3<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV3<L>, rng: &mut R) -> Self {
        Builder::from_bytes(rng.gen())
        .set_variant(Variant::RFC4122)
        .set_version(Version::Md5)
        .build()
    }

    fn dummy(UuidV3(_, namespace, name): &UuidV3<L>) -> Self {
        Self::new_v3(namespace, name)
    }
}

impl<L: Data> Dummy<UuidV3<L>> for String {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV3<L>, rng: &mut R) -> Self {
        Uuid::dummy_with_rng(config, rng).to_hyphenated().to_string()
    }

    fn dummy(config: &UuidV3<L>) -> Self {
        Uuid::dummy(config).to_hyphenated().to_string()
    }
}

#[allow(unused_variables)]
impl<L: Data> Dummy<UuidV4<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV4<L>, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(_: &UuidV4<L>) -> Self {
        Self::new_v4()
    }
}

#[allow(unused_variables)]
impl<L: Data> Dummy<UuidV5<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV5<L>, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(UuidV5(_, namespace, name): &UuidV5<L>) -> Self {
        Self::new_v5(namespace, name)
    }
}

#[allow(unused_variables)]
impl<L: Data> Dummy<UuidSeed<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidSeed<L>, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(UuidSeed(_, seed): &UuidSeed<L>) -> Self {
        Self::from_slice(&seed).expect("generate uuid::Uuid::from_slice")
    }
}

#[allow(unused_variables)]
impl Dummy<Faker> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &Faker, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(config: &Faker) -> Self {
        Self::new_v4()
    }
}<commit_msg>mark dummy_with_rng for v4 as unimplemented, since it's non-deterministic<commit_after>use uuid::{Builder, Uuid, Variant, Version};

use crate::{Dummy, Faker};
use crate::faker::uuid::raw::*;
use crate::locales::Data;

impl<L: Data> Dummy<UuidV1<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV1<L>, rng: &mut R) -> Self {
      let bytes: [u8;8] = rng.gen();
      let (ticks, _) = config.1.to_rfc4122();
      let time_low = (ticks & 0xFFFF_FFFF) as u32;
      let time_mid = ((ticks >> 32) & 0xFFFF) as u16;
      let time_high_and_version = (((ticks >> 48) & 0x0FFF) as u16) | (1 << 12);
      Uuid::from_fields(time_low, time_mid, time_high_and_version, &bytes).expect("generate uuid::Uuid::from_fields")
    }

    fn dummy(UuidV1(_, timestamp, node_id): &UuidV1<L>) -> Self {
        Self::new_v1(*timestamp, node_id).expect("generate uuid::Uuid::new_v1")
    }
}

impl<L: Data> Dummy<UuidV1<L>> for String {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV1<L>, rng: &mut R) -> Self {
        Uuid::dummy_with_rng(config, rng).to_hyphenated().to_string()
    }

    fn dummy(config: &UuidV1<L>) -> Self {
        Uuid::dummy(config).to_hyphenated().to_string()
    }
}

impl<L: Data> Dummy<UuidV3<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV3<L>, rng: &mut R) -> Self {
        Builder::from_bytes(rng.gen())
        .set_variant(Variant::RFC4122)
        .set_version(Version::Md5)
        .build()
    }

    fn dummy(UuidV3(_, namespace, name): &UuidV3<L>) -> Self {
        Self::new_v3(namespace, name)
    }
}

impl<L: Data> Dummy<UuidV3<L>> for String {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV3<L>, rng: &mut R) -> Self {
        Uuid::dummy_with_rng(config, rng).to_hyphenated().to_string()
    }

    fn dummy(config: &UuidV3<L>) -> Self {
        Uuid::dummy(config).to_hyphenated().to_string()
    }
}

impl<L: Data> Dummy<UuidV4<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(_: &UuidV4<L>, _: &mut R) -> Self {
        unimplemented!()
    }

    fn dummy(_: &UuidV4<L>) -> Self {
        Self::new_v4()
    }
}

#[allow(unused_variables)]
impl<L: Data> Dummy<UuidV5<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidV5<L>, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(UuidV5(_, namespace, name): &UuidV5<L>) -> Self {
        Self::new_v5(namespace, name)
    }
}

#[allow(unused_variables)]
impl<L: Data> Dummy<UuidSeed<L>> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &UuidSeed<L>, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(UuidSeed(_, seed): &UuidSeed<L>) -> Self {
        Self::from_slice(&seed).expect("generate uuid::Uuid::from_slice")
    }
}

#[allow(unused_variables)]
impl Dummy<Faker> for Uuid {
    fn dummy_with_rng<R: rand::Rng + ?Sized>(config: &Faker, rng: &mut R) -> Self {
        todo!()
    }

    fn dummy(config: &Faker) -> Self {
        Self::new_v4()
    }
}<|endoftext|>


<commit_before>use std::mem;
use std::marker::PhantomData;
use super::SessionType;
use protocol::{Channel, Protocol, Handler, Defer, DeferFunc};

/// This trait effectively posits that a protocol which handles `T` must
/// additionally handle other types. If `T` is an `Accept<S, Q>` the
/// protocol must handle `S` *and* be an `Acceptor` of `Q`. If `T` is
/// a `Finally<S>` it must handle `S`.
pub trait Acceptor<I, E: SessionType, T>: Protocol + Sized {
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, usize) -> Defer<Self, I>;
}
impl<I, E: SessionType, H: Protocol + Handler<I, E, S> + Acceptor<I, E, Q>, S: SessionType, Q: SessionType> Acceptor<I, E, Accept<S, Q>> for H {
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, num: usize) -> Defer<H, I> {
		if num == 0 {
			let next_func: DeferFunc<Self, I, E, S> = <Self as Handler<I, E, S>>::with;

			Defer::new(chan, unsafe { mem::transmute(next_func) }, true)
		} else {
			<Self as Acceptor<I, E, Q>>::defer(chan, num - 1)
		}
	}
}
impl<I, E: SessionType, H: Protocol + Handler<I, E, S>,                     S: SessionType>                 Acceptor<I, E, Finally<S>>   for H {
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, _: usize) -> Defer<H, I> {
		// regardless of num we cannot proceed further than Finally
		let next_func: DeferFunc<Self, I, E, S> = <Self as Handler<I, E, S>>::with;

		Defer::new(chan, unsafe { mem::transmute(next_func) }, true)
	}
}

/// Choose from `S` or something in `Q`.
pub struct Choose<S: SessionType, Q: SessionType>(PhantomData<(S, Q)>);

unsafe impl<S: SessionType, Q: SessionType> SessionType for Choose<S, Q> {
	type Dual = Accept<S::Dual, Q::Dual>;
}

trait NotSame { }
impl NotSame for .. { }
impl<A> !NotSame for (A, A) { }

/// This trait selects for the de-Bruijn index of a protocol embedded within
/// a `Choose` decision tree.
pub trait Chooser<T> {
	fn num() -> usize;
}

impl<S: SessionType, Q: SessionType> Chooser<S> for Choose<S, Q> {
	fn num() -> usize { 0 }
}

impl<S: SessionType> Chooser<S> for Finally<S> {
	fn num() -> usize { 0 }
}

impl<R: SessionType, S: SessionType, Q: SessionType + Chooser<S>> Chooser<S> for Choose<R, Q>
	where (S, R): NotSame
{
	fn num() -> usize { Q::num().checked_add(1).unwrap() }
}

/// Accept either `S` or something in `Q`.
pub struct Accept<S: SessionType, Q: SessionType>(PhantomData<(S, Q)>);

unsafe impl<S: SessionType, Q: SessionType> SessionType for Accept<S, Q> {
	type Dual = Choose<S::Dual, Q::Dual>;
}

/// Finally choose `S`.
pub struct Finally<S: SessionType>(PhantomData<S>);

unsafe impl<S: SessionType> SessionType for Finally<S> {
	type Dual = Finally<S::Dual>;
}

#[test]
fn check_choose_works() {
	use super::{Recv, End};

	type GetUsize = Recv<usize, End>;
	type GetU8 = Recv<u8, End>;
	type GetString = Recv<String, End>;
	type Getisize = Recv<isize, End>;

	type Proto = Choose<Getisize, Choose<GetString, Choose<GetU8, Finally<GetUsize>>>>;

	assert_eq!(<Proto as Chooser<Getisize>>::num(), 0);
	assert_eq!(<Proto as Chooser<GetString>>::num(), 1);
	assert_eq!(<Proto as Chooser<GetU8>>::num(), 2);
	assert_eq!(<Proto as Chooser<GetUsize>>::num(), 3);
}<commit_msg>Add always inlining to Chooser/Acceptor.<commit_after>use std::mem;
use std::marker::PhantomData;
use super::SessionType;
use protocol::{Channel, Protocol, Handler, Defer, DeferFunc};

/// This trait effectively posits that a protocol which handles `T` must
/// additionally handle other types. If `T` is an `Accept<S, Q>` the
/// protocol must handle `S` *and* be an `Acceptor` of `Q`. If `T` is
/// a `Finally<S>` it must handle `S`.
pub trait Acceptor<I, E: SessionType, T>: Protocol + Sized {
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, usize) -> Defer<Self, I>;
}
impl<I, E: SessionType, H: Protocol + Handler<I, E, S> + Acceptor<I, E, Q>, S: SessionType, Q: SessionType> Acceptor<I, E, Accept<S, Q>> for H {
	#[inline(always)]
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, num: usize) -> Defer<H, I> {
		if num == 0 {
			let next_func: DeferFunc<Self, I, E, S> = <Self as Handler<I, E, S>>::with;

			Defer::new(chan, unsafe { mem::transmute(next_func) }, true)
		} else {
			<Self as Acceptor<I, E, Q>>::defer(chan, num - 1)
		}
	}
}
impl<I, E: SessionType, H: Protocol + Handler<I, E, S>,                     S: SessionType>                 Acceptor<I, E, Finally<S>>   for H {
	#[inline(always)]
	fn defer<Y: SessionType>(chan: Channel<Self, I, E, Y>, _: usize) -> Defer<H, I> {
		// regardless of num we cannot proceed further than Finally
		let next_func: DeferFunc<Self, I, E, S> = <Self as Handler<I, E, S>>::with;

		Defer::new(chan, unsafe { mem::transmute(next_func) }, true)
	}
}

/// Choose from `S` or something in `Q`.
pub struct Choose<S: SessionType, Q: SessionType>(PhantomData<(S, Q)>);

unsafe impl<S: SessionType, Q: SessionType> SessionType for Choose<S, Q> {
	type Dual = Accept<S::Dual, Q::Dual>;
}

trait NotSame { }
impl NotSame for .. { }
impl<A> !NotSame for (A, A) { }

/// This trait selects for the de-Bruijn index of a protocol embedded within
/// a `Choose` decision tree.
pub trait Chooser<T> {
	fn num() -> usize;
}

impl<S: SessionType, Q: SessionType> Chooser<S> for Choose<S, Q> {
	#[inline(always)]
	fn num() -> usize { 0 }
}

impl<S: SessionType> Chooser<S> for Finally<S> {
	#[inline(always)]
	fn num() -> usize { 0 }
}

impl<R: SessionType, S: SessionType, Q: SessionType + Chooser<S>> Chooser<S> for Choose<R, Q>
	where (S, R): NotSame
{
	#[inline(always)]
	fn num() -> usize { Q::num().checked_add(1).unwrap() }
}

/// Accept either `S` or something in `Q`.
pub struct Accept<S: SessionType, Q: SessionType>(PhantomData<(S, Q)>);

unsafe impl<S: SessionType, Q: SessionType> SessionType for Accept<S, Q> {
	type Dual = Choose<S::Dual, Q::Dual>;
}

/// Finally choose `S`.
pub struct Finally<S: SessionType>(PhantomData<S>);

unsafe impl<S: SessionType> SessionType for Finally<S> {
	type Dual = Finally<S::Dual>;
}

#[test]
fn check_choose_works() {
	use super::{Recv, End};

	type GetUsize = Recv<usize, End>;
	type GetU8 = Recv<u8, End>;
	type GetString = Recv<String, End>;
	type Getisize = Recv<isize, End>;

	type Proto = Choose<Getisize, Choose<GetString, Choose<GetU8, Finally<GetUsize>>>>;

	assert_eq!(<Proto as Chooser<Getisize>>::num(), 0);
	assert_eq!(<Proto as Chooser<GetString>>::num(), 1);
	assert_eq!(<Proto as Chooser<GetU8>>::num(), 2);
	assert_eq!(<Proto as Chooser<GetUsize>>::num(), 3);
}<|endoftext|>


<commit_before>extern crate chrono;
extern crate futures;
extern crate hyper;
extern crate openssl_probe;
extern crate select;
extern crate telegram_bot;
extern crate tokio;
extern crate tokio_core;

// For discord
use chrono::Timelike;

// For file reading
use std::fs::File;
use std::fs::OpenOptions;

use std::io::Read;
use std::io::Write;

// For multithreading
use std::ops::DerefMut;

use std::sync::{Arc, Mutex};
use std::thread;

// For telegram
use futures::Stream;
use telegram_bot::*;

use hyper::rt::{Future, Stream as HyperStream};
use hyper::Client;

use tokio_core::reactor::Core;

#[derive(Debug, Clone)]
enum TelegramChannel {
    ChannelId(telegram_bot::types::ChannelId),
    ChatMessage(telegram_bot::types::MessageChat),
}

impl TelegramChannel {
    fn to_chat_ref(&self) -> ChatRef {
        match self {
            TelegramChannel::ChannelId(id) => id.to_chat_ref(),
            TelegramChannel::ChatMessage(msg) => msg.to_chat_ref(),
        }
    }
}

impl TelegramChannel {
    fn send_message(&self, message: String, api: &Api) {
        let send_message = self.to_chat_ref().text(message);
        api.spawn(send_message);
    }
}

type ResponseCode = hyper::StatusCode;

fn send_get(url: String) -> (String, ResponseCode) {
    let url = url.replace(" ", "%20");

    let body_mutex = Arc::new(Mutex::new(None));
    let body_mutex_c = body_mutex.clone();

    let status_mutex = Arc::new(Mutex::new(None));
    let status_mutex_c = status_mutex.clone();

    let t = thread::spawn(move || {
        tokio::run({
            let client = Client::new();
            dbg!(url.clone());
            client
                .get(url.parse().unwrap())
                .and_then(move |res| {
                    *status_mutex_c.lock().unwrap() = Some(res.status());
                    // asynchronously concatenate chunks of the body
                    res.into_body().concat2()
                })
                .map_err(|err| {
                    println!("Error: {}", err);
                })
                .and_then(move |body| {
                    *body_mutex_c.lock().unwrap() =
                        Some(format!("{}", std::str::from_utf8(&body).unwrap()));
                    Ok(())
                })
        });
    });

    t.join().unwrap();
    let body = format!("{}", (body_mutex.lock().unwrap().as_ref().unwrap()));
    let unwrapped_status_mutex = status_mutex.lock().unwrap();
    let status = unwrapped_status_mutex.as_ref().unwrap();
    (body, *status)
}

fn check_food(food: String) -> String {
    send_get(format!("http://localhost:8000/food?food={}", food)).0
}

// Get the telegram token file from memory
fn load_telegram_token() -> String {
    let mut token = String::new();
    let _ = File::open("telegram_token")
        .expect("No token file")
        .read_to_string(&mut token);
    token.trim().to_string()
}

// Generic user that should work across Discord and Telegram
struct User {
    id: UserId,
    // UniqueName is "first_name last_name (username)";
    //  We are only guarenteed to have "first_name" though.
    // It is not constant for a user
    unique_name: String,
    // Whether this user is an admin of the bot
    is_owner: bool,
}

impl User {
    fn from_telegram_message(user: telegram_bot::types::User) -> User {
        let full_name = match (user.last_name, user.username) {
            (None, None) => user.first_name.clone(),
            (None, Some(username)) => format!("{} ({})", user.first_name, username),
            (Some(last_name), None) => format!("{} {}", user.first_name, last_name),
            (Some(last_name), Some(username)) => {
                format!("{} {} ({})", user.first_name, last_name, username)
            }
        };

        User {
            id: user.id,
            unique_name: full_name,
            is_owner: user.id == telegram_bot::types::UserId::new(698_919_547),
        }
    }
}

fn handle_message(
    content: String,
    author: User,
    channel: TelegramChannel,
    listeners: Arc<Mutex<Vec<(TelegramChannel, String)>>>,
    telegram_api: &Api,
) {
    println!("{}: {} says: {}", author.unique_name, author.id, content);
    if !content.starts_with('!') && !content.starts_with('/') {
        // It's not a command, so we don't care about it
        return;
    }

    if content.starts_with("/menu ") {
        let item: &str = &content[6..];

        let response = dbg!(check_food(item.to_string()));

        channel.send_message(format!("{}", response), &telegram_api);
    } else if content.starts_with("/echo ") {
        let input: String = content[6..].to_string();

        let res = send_get(format!("http://localhost:8000/echo?input={}", input)).0;

        channel.send_message(format!("{}", res), &telegram_api);
    } else if content.starts_with("/register ") {
        let item: String = content[10..].to_string();
        listeners
            .lock()
            .unwrap()
            .deref_mut()
            .push((channel.clone(), item.clone()));
        save_listeners(listeners.lock().unwrap().deref_mut());
        channel.send_message(
            format!("Will check for {}", item).to_string(),
            &telegram_api,
        );

        let response = check_food(item.to_string());

        channel.send_message(format!("{}", response), &telegram_api);
    } else if content == "/help" {
        channel.send_message(
            "/menu [food name] => tells you where that food is being served today".to_string(),
            &telegram_api,
        );

        channel.send_message(
                    "/register [food name] => schedules it to tell you each day where that food is being served that day"
                        .to_string(),
                    &telegram_api,
                );
    } else if content.starts_with("/room ") {
        let room: String = content[6..].to_string();

        let (_, status_code) = send_get(format!("http://localhost:8000/room/?room={}", room));

        if status_code == 200 {
            channel.send_message(format!("Rooms found"), &telegram_api);
        } else {
            channel.send_message(format!("No rooms found"), &telegram_api);
        }
    } else if content == "/run" {
        channel.send_message(
            "Checking for preregistered foods".to_string(),
            &telegram_api,
        );
        check_for_foods(&listeners, &telegram_api);
    } else if content.starts_with("/quit") && author.is_owner {
        channel.send_message("UMass Bot Quitting".to_string(), &telegram_api);
        std::process::exit(0);
    }
}

fn read_listeners() -> Vec<(TelegramChannel, String)> {
    let mut listeners_string: String = String::new();
    let _ = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .open("telegram_listeners.txt")
        .expect("No telegram listeners file")
        .read_to_string(&mut listeners_string);

    let mut listeners: Vec<(TelegramChannel, String)> = vec![];

    for line in listeners_string.split('\n') {
        if line == "" {
            continue;
        }
        let sections: Vec<&str> = line.split(' ').collect();
        let app = sections[0];
        if app == "telegram" {
            let id = TelegramChannel::ChannelId(telegram_bot::types::ChannelId::from(
                sections[1]
                    .parse::<i64>()
                    .expect("Couldn\'t parse channel id"),
            ));

            let food: String = sections[2..].join(" ").to_string();
            listeners.push((id, food));
        }
    }

    listeners
}

fn save_listeners(pairs: &[(TelegramChannel, String)]) {
    let mut listeners_string: String = String::new();
    pairs.iter().for_each(|x| {
        listeners_string = match *x {
            (ref id, ref food) => format!("{}telegram {:?} {}\n", listeners_string, id, food),
        };
    });

    let listeners_string = listeners_string.trim();

    let _ = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .truncate(true)
        .open("telegram_listeners.txt")
        .unwrap()
        .write_all(listeners_string.as_bytes());
}

// Runs at 6 AM in summer or 5 AM in winter
fn get_time_till_scheduled() -> std::time::Duration {
    let current_time_utc = chrono::prelude::Utc::now();
    let current_time: chrono::DateTime<chrono::offset::FixedOffset> = chrono::DateTime::from_utc(
        current_time_utc.naive_utc(),
        chrono::offset::FixedOffset::west(4 * 60 * 60),
        // Four hours west of the date line
        // Four instead of five because 5am/6am is a better default than 6am/7am
    );
    let next_run_date = if current_time.time().hour() < 6
        || (current_time.hour() == 6 && current_time.minute() < 5)
    {
        // We want to do it today (in UTC) if it is still yesterday in Eastern Time
        current_time
    } else {
        current_time + chrono::Duration::days(1)
    }
    .date();

    let next_run = next_run_date.and_hms(6, 5, 0);

    (next_run - current_time).to_std().unwrap()
}

fn check_for_foods(listeners: &Arc<Mutex<Vec<(TelegramChannel, String)>>>, telegram_api: &Api) {
    listeners
        .lock()
        .unwrap()
        .to_vec()
        .into_iter()
        .for_each(|(channel, food)| {
            println!("Checking on {:?} for {}", channel, food);
            let response = check_food(food);

            channel.send_message(format!("{}", response), telegram_api);
        });
}

fn main() {
    // Allow openssl crosscompiling to work
    openssl_probe::init_ssl_cert_env_vars();

    let listeners: Arc<Mutex<Vec<(TelegramChannel, String)>>> =
        Arc::new(Mutex::new(read_listeners()));

    let telegram_token = load_telegram_token();

    // Listeners loop
    /*let listeners_clone = Arc::clone(&listeners);
    let telegram_token_clone = telegram_token.clone();
    thread::spawn(move || {
        let listeners = listeners_clone;
        loop {
            let telegram_token = telegram_token_clone.clone();
            println!("Seconds till scheduled: {:?}", get_time_till_scheduled());
            thread::sleep(get_time_till_scheduled());
            println!("Checking for foods now!");
            check_for_foods(&listeners, &telegram_token);
        }
    });*/

    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let api = Api::configure(telegram_token.clone())
        .build(handle)
        .unwrap();

    let stream = api.stream().then(|mb_update| {
        let res: Result<Result<Update, Error>, ()> = Ok(mb_update);
        res
    });

    // Fetch new updates via long poll method
    let future = stream.for_each(move |update| {
        match update {
            Ok(update) => {
                // If the received update contains a new message...
                if let UpdateKind::Message(message) = update.kind {
                    println!("Some message {:?}", message);
                    if let MessageKind::Text { ref data, .. } = message.kind {
                        handle_message(
                            data.to_string(),
                            User::from_telegram_message(message.from),
                            TelegramChannel::ChatMessage(message.chat),
                            Arc::clone(&listeners),
                            &api,
                        );
                    }
                } else {
                    println!("Some update {:?}", update);
                }
            }
            Err(e) => {
                println!("Some error {:?}", e);
            }
        }

        Ok(())
    });

    core.run(future).unwrap();
}
<commit_msg>Fix name of variable<commit_after>extern crate chrono;
extern crate futures;
extern crate hyper;
extern crate openssl_probe;
extern crate select;
extern crate telegram_bot;
extern crate tokio;
extern crate tokio_core;

// For discord
use chrono::Timelike;

// For file reading
use std::fs::File;
use std::fs::OpenOptions;

use std::io::Read;
use std::io::Write;

// For multithreading
use std::ops::DerefMut;

use std::sync::{Arc, Mutex};
use std::thread;

// For telegram
use futures::Stream;
use telegram_bot::*;

use hyper::rt::{Future, Stream as HyperStream};
use hyper::Client;

use tokio_core::reactor::Core;

#[derive(Debug, Clone)]
enum TelegramChannel {
    ChannelId(telegram_bot::types::ChannelId),
    ChatMessage(telegram_bot::types::MessageChat),
}

impl TelegramChannel {
    fn to_chat_ref(&self) -> ChatRef {
        match self {
            TelegramChannel::ChannelId(id) => id.to_chat_ref(),
            TelegramChannel::ChatMessage(msg) => msg.to_chat_ref(),
        }
    }
}

impl TelegramChannel {
    fn send_message(&self, message: String, api: &Api) {
        let send_message = self.to_chat_ref().text(message);
        api.spawn(send_message);
    }
}

type ResponseCode = hyper::StatusCode;

fn send_get(url: String) -> (String, ResponseCode) {
    let url = url.replace(" ", "%20");

    let body_mutex = Arc::new(Mutex::new(None));
    let body_mutex_c = body_mutex.clone();

    let status_mutex = Arc::new(Mutex::new(None));
    let status_mutex_c = status_mutex.clone();

    let t = thread::spawn(move || {
        tokio::run({
            let client = Client::new();
            dbg!(url.clone());
            client
                .get(url.parse().unwrap())
                .and_then(move |res| {
                    *status_mutex_c.lock().unwrap() = Some(res.status());
                    // asynchronously concatenate chunks of the body
                    res.into_body().concat2()
                })
                .map_err(|err| {
                    println!("Error: {}", err);
                })
                .and_then(move |body| {
                    *body_mutex_c.lock().unwrap() =
                        Some(format!("{}", std::str::from_utf8(&body).unwrap()));
                    Ok(())
                })
        });
    });

    t.join().unwrap();
    let body = format!("{}", (body_mutex.lock().unwrap().as_ref().unwrap()));
    let unwrapped_status_mutex = status_mutex.lock().unwrap();
    let status = unwrapped_status_mutex.as_ref().unwrap();
    (body, *status)
}

fn check_food(food: String) -> String {
    send_get(format!("http://localhost:8000/food?food={}", food)).0
}

// Get the telegram token file from memory
fn load_telegram_token() -> String {
    let mut token = String::new();
    let _ = File::open("telegram_token")
        .expect("No token file")
        .read_to_string(&mut token);
    token.trim().to_string()
}

// Generic user that should work across Discord and Telegram
struct User {
    id: UserId,
    // UniqueName is "first_name last_name (username)";
    //  We are only guarenteed to have "first_name" though.
    // It is not constant for a user
    unique_name: String,
    // Whether this user is an admin of the bot
    is_owner: bool,
}

impl User {
    fn from_telegram_message(user: telegram_bot::types::User) -> User {
        let full_name = match (user.last_name, user.username) {
            (None, None) => user.first_name.clone(),
            (None, Some(username)) => format!("{} ({})", user.first_name, username),
            (Some(last_name), None) => format!("{} {}", user.first_name, last_name),
            (Some(last_name), Some(username)) => {
                format!("{} {} ({})", user.first_name, last_name, username)
            }
        };

        User {
            id: user.id,
            unique_name: full_name,
            is_owner: user.id == telegram_bot::types::UserId::new(698_919_547),
        }
    }
}

fn handle_message(
    content: String,
    author: User,
    channel: TelegramChannel,
    listeners: Arc<Mutex<Vec<(TelegramChannel, String)>>>,
    telegram_api: &Api,
) {
    println!("{}: {} says: {}", author.unique_name, author.id, content);
    if !content.starts_with('!') && !content.starts_with('/') {
        // It's not a command, so we don't care about it
        return;
    }

    if content.starts_with("/menu ") {
        let item: &str = &content[6..];

        let response = dbg!(check_food(item.to_string()));

        channel.send_message(format!("{}", response), &telegram_api);
    } else if content.starts_with("/echo ") {
        let input: String = content[6..].to_string();

        let res = send_get(format!("http://localhost:8000/echo?input={}", input)).0;

        channel.send_message(format!("{}", res), &telegram_api);
    } else if content.starts_with("/register ") {
        let item: String = content[10..].to_string();
        listeners
            .lock()
            .unwrap()
            .deref_mut()
            .push((channel.clone(), item.clone()));
        save_listeners(listeners.lock().unwrap().deref_mut());
        channel.send_message(
            format!("Will check for {}", item).to_string(),
            &telegram_api,
        );

        let response = check_food(item.to_string());

        channel.send_message(format!("{}", response), &telegram_api);
    } else if content == "/help" {
        channel.send_message(
            "/menu [food name] => tells you where that food is being served today".to_string(),
            &telegram_api,
        );

        channel.send_message(
                    "/register [food name] => schedules it to tell you each day where that food is being served that day"
                        .to_string(),
                    &telegram_api,
                );
    } else if content.starts_with("/room ") {
        let room: String = content[6..].to_string();

        let (_, status_code) = send_get(format!("http://localhost:8000/room/?room={}", room));

        if status_code == 200 {
            channel.send_message(format!("Rooms found"), &telegram_api);
        } else {
            channel.send_message(format!("No rooms found"), &telegram_api);
        }
    } else if content == "/run" {
        channel.send_message(
            "Checking for preregistered foods".to_string(),
            &telegram_api,
        );
        check_for_foods(&listeners, &telegram_api);
    } else if content.starts_with("/quit") && author.is_owner {
        channel.send_message("UMass Bot Quitting".to_string(), &telegram_api);
        std::process::exit(0);
    }
}

fn read_listeners() -> Vec<(TelegramChannel, String)> {
    let mut listeners_string: String = String::new();
    let _ = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .open("telegram_listeners.txt")
        .expect("No telegram listeners file")
        .read_to_string(&mut listeners_string);

    let mut listeners: Vec<(TelegramChannel, String)> = vec![];

    for line in listeners_string.split('\n') {
        if line == "" {
            continue;
        }
        let sections: Vec<&str> = line.split(' ').collect();
        let app = sections[0];
        if app == "telegram" {
            let id = TelegramChannel::ChannelId(telegram_bot::types::ChannelId::from(
                sections[1]
                    .parse::<i64>()
                    .expect("Couldn\'t parse channel id"),
            ));

            let food: String = sections[2..].join(" ").to_string();
            listeners.push((id, food));
        }
    }

    listeners
}

fn save_listeners(pairs: &[(TelegramChannel, String)]) {
    let mut listeners_string: String = String::new();
    pairs.iter().for_each(|x| {
        listeners_string = match *x {
            (ref id, ref food) => format!("{}telegram {:?} {}\n", listeners_string, id, food),
        };
    });

    let listeners_string = listeners_string.trim();

    let _ = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .truncate(true)
        .open("telegram_listeners.txt")
        .unwrap()
        .write_all(listeners_string.as_bytes());
}

// Runs at 6 AM in summer or 5 AM in winter
fn get_time_till_scheduled() -> std::time::Duration {
    let current_time_utc = chrono::prelude::Utc::now();
    let current_time: chrono::DateTime<chrono::offset::FixedOffset> = chrono::DateTime::from_utc(
        current_time_utc.naive_utc(),
        chrono::offset::FixedOffset::west(4 * 60 * 60),
        // Four hours west of the date line
        // Four instead of five because 5am/6am is a better default than 6am/7am
    );
    let next_run_date = if current_time.time().hour() < 6
        || (current_time.hour() == 6 && current_time.minute() < 5)
    {
        // We want to do it today (in UTC) if it is still yesterday in Eastern Time
        current_time
    } else {
        current_time + chrono::Duration::days(1)
    }
    .date();

    let next_run = next_run_date.and_hms(6, 5, 0);

    (next_run - current_time).to_std().unwrap()
}

fn check_for_foods(listeners: &Arc<Mutex<Vec<(TelegramChannel, String)>>>, telegram_api: &Api) {
    listeners
        .lock()
        .unwrap()
        .to_vec()
        .into_iter()
        .for_each(|(channel, food)| {
            println!("Checking on {:?} for {}", channel, food);
            let response = check_food(food);

            channel.send_message(format!("{}", response), telegram_api);
        });
}

fn main() {
    // Allow openssl crosscompiling to work
    openssl_probe::init_ssl_cert_env_vars();

    let listeners: Arc<Mutex<Vec<(TelegramChannel, String)>>> =
        Arc::new(Mutex::new(read_listeners()));

    let telegram_token = load_telegram_token();

    // Listeners loop
    /*let listeners_clone = Arc::clone(&listeners);
    let telegram_token_clone = telegram_token.clone();
    thread::spawn(move || {
        let listeners = listeners_clone;
        loop {
            let telegram_token = telegram_token_clone.clone();
            println!("Seconds till scheduled: {:?}", get_time_till_scheduled());
            thread::sleep(get_time_till_scheduled());
            println!("Checking for foods now!");
            check_for_foods(&listeners, &api);
        }
    });*/

    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let api = Api::configure(telegram_token.clone())
        .build(handle)
        .unwrap();

    let stream = api.stream().then(|mb_update| {
        let res: Result<Result<Update, Error>, ()> = Ok(mb_update);
        res
    });

    // Fetch new updates via long poll method
    let future = stream.for_each(move |update| {
        match update {
            Ok(update) => {
                // If the received update contains a new message...
                if let UpdateKind::Message(message) = update.kind {
                    println!("Some message {:?}", message);
                    if let MessageKind::Text { ref data, .. } = message.kind {
                        handle_message(
                            data.to_string(),
                            User::from_telegram_message(message.from),
                            TelegramChannel::ChatMessage(message.chat),
                            Arc::clone(&listeners),
                            &api,
                        );
                    }
                } else {
                    println!("Some update {:?}", update);
                }
            }
            Err(e) => {
                println!("Some error {:?}", e);
            }
        }

        Ok(())
    });

    core.run(future).unwrap();
}
<|endoftext|>


<commit_before>use cgmath::Point3;
use noise::{Seed, Brownian2, Brownian3, perlin2, perlin3};

use voxel;

pub struct T {
  pub height: Brownian2<f64, fn (&Seed, &[f64; 2]) -> f64>,
  pub features: Brownian3<f64, fn (&Seed, &[f64; 3]) -> f64>,
  pub seed: Seed,
}

impl T {
  pub fn new(seed: Seed) -> T {
    let perlin2: fn(&Seed, &[f64; 2]) -> f64 = perlin2;
    let perlin3: fn(&Seed, &[f64; 3]) -> f64 = perlin3;
    T {
      seed: seed,
      height:
        Brownian2::new(perlin2, 4)
        .frequency(1.0 / 8.0)
        .persistence(8.0)
        .lacunarity(1.0 / 4.0)
      ,
      features:
        Brownian3::new(perlin3, 2)
        .frequency(1.0 / 32.0)
        .persistence(8.0)
        .lacunarity(1.0 / 4.0)
      ,
    }
  }
}

impl voxel::field::T for T {
  /// The height of the field at a given x,y,z.
  fn density_at(this: &Self, p: &Point3<f32>) -> f32 {
    let height = this.height.apply(&this.seed, &[p.x as f64, p.z as f64]);
    let height = height as f32;
    let heightmap_density = height - p.y;

    let feature_density = this.features.apply(&this.seed, &[p.x as f64, p.y as f64, p.z as f64]) * 8.0;
    let feature_density = feature_density as f32;

    heightmap_density + feature_density
  }
}
<commit_msg>tweak terrain generation<commit_after>use cgmath::Point3;
use noise::{Seed, Brownian2, Brownian3, perlin2, perlin3};

use voxel;

pub struct T {
  pub height: Brownian2<f64, fn (&Seed, &[f64; 2]) -> f64>,
  pub features: Brownian3<f64, fn (&Seed, &[f64; 3]) -> f64>,
  pub seed: Seed,
}

impl T {
  pub fn new(seed: Seed) -> T {
    let perlin2: fn(&Seed, &[f64; 2]) -> f64 = perlin2;
    let perlin3: fn(&Seed, &[f64; 3]) -> f64 = perlin3;
    T {
      seed: seed,
      height:
        Brownian2::new(perlin2, 5)
        .frequency(1.0 / 4.0)
        .persistence(2.0)
        .lacunarity(1.0 / 2.0)
      ,
      features:
        Brownian3::new(perlin3, 2)
        .frequency(1.0 / 32.0)
        .persistence(8.0)
        .lacunarity(1.0 / 4.0)
      ,
    }
  }
}

impl voxel::field::T for T {
  /// The height of the field at a given x,y,z.
  fn density_at(this: &Self, p: &Point3<f32>) -> f32 {
    let height = this.height.apply(&this.seed, &[p.x as f64, p.z as f64]);
    let height = height as f32;
    let heightmap_density = height - p.y;

    let feature_density = this.features.apply(&this.seed, &[p.x as f64, p.y as f64, p.z as f64]) * 8.0;
    let feature_density = feature_density as f32;

    heightmap_density + feature_density
  }
}
<|endoftext|>


<commit_before>extern crate rusttype;
use self::rusttype::{FontCollection, Scale, point, PositionedGlyph};

/// Returns the brightness value between 0 and 255 for the input character.
///
/// Based on the algorithm described [here](http://mattmik.com/articles/ascii/ascii.html).
/// The string is rendered in the
/// [Inconsolata font](http://www.levien.com/type/myfonts/inconsolata.html) for the
/// purposes of determining the brightness.
pub fn calculate_character_brightness(c: char) -> i32 {
    let string = c.to_string();
    // Generate an in-memory bitmap image of the character
    // Code snippet taken from https://github.com/dylanede/rusttype/blob/master/examples/simple.rs
    let font_data = include_bytes!("Inconsolata-Regular.ttf");
    let collection = FontCollection::from_bytes(font_data as &[u8]);
    let font = collection.into_font().unwrap();

    let height: f32 = 50.0;
    let width = 50;
    let pixel_height = height.ceil() as usize;

    let scale = Scale { x: height * 2.0, y: height };

    // The origin of a line of text is at the baseline (roughly where non-descending letters sit).
    // We don't want to clip the text, so we shift it down with an offset when laying it out.
    // v_metrics.ascent is the distance between the baseline and the highest edge of any glyph in
    // the font. That's enough to guarantee that there's no clipping.
    let v_metrics = font.v_metrics(scale);
    let offset = point(0.0, v_metrics.ascent);

    let glyphs: Vec<PositionedGlyph> = font.layout(&string, scale, offset).collect();
    let glyph: PositionedGlyph = glyphs.last().unwrap().standalone();

    // Represents the character as a vector of "pixel coverage" values
    let mut pixel_data = vec![0.0; width * pixel_height];

    // The `v` parameter to the closure in glyph.draw() is a "pixel coverage" value from 0 to 1
    if let Some(bb) = glyph.pixel_bounding_box() {
        glyph.draw(|x, y, v| {
            let positive_v = if v < 0.0 { 0.0 } else { v };
            let scaled_v = positive_v * 255.0;
            let x = x as i32 + bb.min.x;
            let y = y as i32 + bb.min.y;
            if x >= 0 && x < width as i32 &&y >= 0 && y < pixel_height as i32 {
                let x = x as usize;
                let y = y as usize;
                pixel_data[(x + y * width)] = scaled_v;
            }
        })
    }

    let total_brightness = pixel_data.iter().fold(0.0, |mut sum, &value| {sum += value; sum});
    let num_pixels = pixel_data.len() as f32;

    if total_brightness == 0.0 || num_pixels == 0.0 {
        0
    }
    else {
        (total_brightness/num_pixels) as i32
    }
}
<commit_msg>Add working test for calculate_character_brightness<commit_after>extern crate rusttype;
use self::rusttype::{FontCollection, Scale, point, PositionedGlyph};

/// Returns the brightness value between 0 and 255 for the input character.
///
/// Based on the algorithm described [here](http://mattmik.com/articles/ascii/ascii.html).
/// The string is rendered in the
/// [Inconsolata font](http://www.levien.com/type/myfonts/inconsolata.html) for the
/// purposes of determining the brightness.
///
/// # Example
/// ```
/// use txtpic_lib::character_set::calculate_character_brightness::calculate_character_brightness;
///
/// let c0 = ' ';
/// let c1 = 'M';
///
/// let b0 = calculate_character_brightness(c0);
/// assert_eq!(b0, 0);
///
/// let b1 = calculate_character_brightness(c1);
/// assert_eq!(b1, 54);
pub fn calculate_character_brightness(c: char) -> i32 {
    let string = c.to_string();
    // Generate an in-memory bitmap image of the character
    // Code snippet taken from https://github.com/dylanede/rusttype/blob/master/examples/simple.rs
    let font_data = include_bytes!("Inconsolata-Regular.ttf");
    let collection = FontCollection::from_bytes(font_data as &[u8]);
    let font = collection.into_font().unwrap();

    let height: f32 = 50.0;
    let width = 50;
    let pixel_height = height.ceil() as usize;

    let scale = Scale { x: height * 2.0, y: height };

    // The origin of a line of text is at the baseline (roughly where non-descending letters sit).
    // We don't want to clip the text, so we shift it down with an offset when laying it out.
    // v_metrics.ascent is the distance between the baseline and the highest edge of any glyph in
    // the font. That's enough to guarantee that there's no clipping.
    let v_metrics = font.v_metrics(scale);
    let offset = point(0.0, v_metrics.ascent);

    let glyphs: Vec<PositionedGlyph> = font.layout(&string, scale, offset).collect();
    let glyph: PositionedGlyph = glyphs.last().unwrap().standalone();

    // Represents the character as a vector of "pixel coverage" values
    let mut pixel_data = vec![0.0; width * pixel_height];

    // The `v` parameter to the closure in glyph.draw() is a "pixel coverage" value from 0 to 1
    if let Some(bb) = glyph.pixel_bounding_box() {
        glyph.draw(|x, y, v| {
            let positive_v = if v < 0.0 { 0.0 } else { v };
            let scaled_v = positive_v * 255.0;
            let x = x as i32 + bb.min.x;
            let y = y as i32 + bb.min.y;
            if x >= 0 && x < width as i32 &&y >= 0 && y < pixel_height as i32 {
                let x = x as usize;
                let y = y as usize;
                pixel_data[(x + y * width)] = scaled_v;
            }
        })
    }

    let total_brightness = pixel_data.iter().fold(0.0, |mut sum, &value| {sum += value; sum});
    let num_pixels = pixel_data.len() as f32;

    if total_brightness == 0.0 || num_pixels == 0.0 {
        0
    }
    else {
        (total_brightness/num_pixels) as i32
    }
}
<|endoftext|>


<commit_before>//! Crate ruma_appservice_api contains serializable types for the requests and responses for each
//! endpoint in the [Matrix](https://matrix.org/) application service API specification. These
//! types can be shared by application service and server code.

#![deny(
    missing_copy_implementations,
    missing_debug_implementations,
    missing_docs
)]
// Since we support Rust 1.34.2, we can't apply this suggestion yet
#![allow(clippy::use_self)]

pub mod v1;
<commit_msg>Remove outdated lint attribute<commit_after>//! Crate ruma_appservice_api contains serializable types for the requests and responses for each
//! endpoint in the [Matrix](https://matrix.org/) application service API specification. These
//! types can be shared by application service and server code.

#![deny(
    missing_copy_implementations,
    missing_debug_implementations,
    missing_docs
)]

pub mod v1;
<|endoftext|>


<commit_before>use memory::page::Page;
use std::mem::{align_of, size_of};
use std::ops::Index;
use std::ptr::write;

#[derive(Copy, Clone)]
pub struct Vector<T: Copy> {
    pub data: *mut T,
    pub length: usize,
}

impl<T: Copy> Vector<T> {
    pub fn new(_page: *mut Page, length: usize) -> Vector<T> {
        unsafe {
            Vector {
                data: (*_page).allocate_raw(length * size_of::<T>(), align_of::<T>()) as *mut T,
                length: length,
            }
        }
    }

    pub fn from_raw_array(_page: *mut Page, array: &[T]) -> Vector<T> {
        let vector = Vector::new(_page, array.len());
        let mut address = vector.data;
        for item in array {
            unsafe {
                write(address, *item);
                address = address.offset(1);
            }
        }
        vector
    }
}

impl<T: Copy> Index<usize> for Vector<T> {
    type Output = T;
    fn index(&self, offset: usize) -> &Self::Output {
        unsafe { &*self.data.offset(offset as isize) }
    }
}

#[test]
fn test_hash_map() {
    use containers::String;
    use io::Console;
    use memory::Heap;
    use memory::Page;
    use memory::Region;
    use memory::StackBucket;
    let mut heap = Heap::create();
    let root_stack_bucket = StackBucket::create(&mut heap);
    let root_page = Page::get(root_stack_bucket as usize);
    let r = Region::create_from_page(root_page);
    let _r_1 = Region::create(&r);
    let vector = Vector::from_raw_array(
        _r_1.page,
        &[
            String::from_string_slice(_r_1.page, "using"),
            String::from_string_slice(_r_1.page, "namespace"),
            String::from_string_slice(_r_1.page, "typedef"),
        ],
    );
    Console::write(&_r_1, vector[0]);
    Console::write(&_r_1, vector[1]);
    Console::write(&_r_1, vector[2]);
}
<commit_msg>vector as Ref in Test<commit_after>use memory::page::Page;
use std::mem::{align_of, size_of};
use std::ops::Index;
use std::ptr::write;

#[derive(Copy, Clone)]
pub struct Vector<T: Copy> {
    pub data: *mut T,
    pub length: usize,
}

impl<T: Copy> Vector<T> {
    pub fn new(_page: *mut Page, length: usize) -> Vector<T> {
        unsafe {
            Vector {
                data: (*_page).allocate_raw(length * size_of::<T>(), align_of::<T>()) as *mut T,
                length: length,
            }
        }
    }

    pub fn from_raw_array(_page: *mut Page, array: &[T]) -> Vector<T> {
        let vector = Vector::new(_page, array.len());
        let mut address = vector.data;
        for item in array {
            unsafe {
                write(address, *item);
                address = address.offset(1);
            }
        }
        vector
    }
}

impl<T: Copy> Index<usize> for Vector<T> {
    type Output = T;
    fn index(&self, offset: usize) -> &Self::Output {
        unsafe { &*self.data.offset(offset as isize) }
    }
}

#[test]
fn test_hash_map() {
    use containers::{Ref, String};
    use io::Console;
    use memory::{Heap, Page, Region, StackBucket};
    let mut heap = Heap::create();
    let root_stack_bucket = StackBucket::create(&mut heap);
    let root_page = Page::get(root_stack_bucket as usize);
    let r = Region::create_from_page(root_page);
    let _r_1 = Region::create(&r);
    let vector = Ref::new(
        _r_1.page,
        Vector::from_raw_array(
            _r_1.page,
            &[
                String::from_string_slice(_r_1.page, "using"),
                String::from_string_slice(_r_1.page, "namespace"),
                String::from_string_slice(_r_1.page, "typedef"),
            ],
        ),
    );
    Console::write(&_r_1, vector[0]);
    Console::write(&_r_1, vector[1]);
    Console::write(&_r_1, vector[2]);
}
<|endoftext|>


<commit_before>use graph::Graph;
use num_cpus;
use script::Environment;
use script::task::Task;
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::rc::Rc;
use std::sync::mpsc::channel;
use std::thread;
use stdlib;

/// A task runner object that holds the state for defined tasks, dependencies, and the scripting
/// runtime.
pub struct Runner {
    /// The script environment.
    environment: Environment,

    /// The current DAG for tasks.
    graph: Graph,

    /// Indicates if actually running tasks should be skipped.
    dry_run: bool,

    /// Indicates if up-to-date tasks should be run anyway.
    always_run: bool,

    /// The number of threads to use.
    jobs: usize,
}

impl Runner {
    /// Creates a new runner instance.
    pub fn new(environment: Environment) -> Runner {
        // By default, set the number of jobs to be one less than the number of available CPU cores.
        let jobs = cmp::max(1, num_cpus::get() - 1);

        Runner {
            environment: environment,
            graph: Graph::new(),
            dry_run: false,
            always_run: false,
            jobs: jobs as usize,
        }
    }

    /// Sets the runner to "dry run" mode.
    ///
    /// When in "dry run" mode, running tasks will operate as normal, except that no task's actions
    /// will be actually run.
    pub fn dry_run(&mut self) {
        self.dry_run = true;
    }

    /// Run all tasks even if they are up-to-date.
    pub fn always_run(&mut self) {
        self.always_run = true;
    }

    /// Sets the number of threads to use to run tasks.
    pub fn jobs(&mut self, jobs: usize) {
        self.jobs = jobs;
    }

    /// Run the default task.
    pub fn run_default(&mut self) -> Result<(), Box<Error>> {
        if let Some(ref name) = self.environment.default_task() {
            let tasks = vec![name];
            self.run(&tasks)
        } else {
            Err("no default task defined".into())
        }
    }

    /// Runs the specified list of tasks.
    ///
    /// Tasks are run in parallel when possible during execution. The maximum number of parallel
    /// jobs can be set with the `jobs()` method.
    pub fn run<S: AsRef<str>>(&mut self, tasks: &[S]) -> Result<(), Box<Error>> {
        // Resolve all tasks given.
        for task in tasks {
            try!(self.resolve_task(task));
        }

        // Determine the schedule of tasks to execute.
        let mut schedule = try!(self.graph.solve(!self.always_run));
        let task_count = schedule.len();
        let thread_count = cmp::min(self.jobs, task_count);

        debug!("running {} task(s) across {} thread(s)", task_count, thread_count);

        // Spawn one thread for each job.
        let mut threads = Vec::new();
        let mut free_threads: HashSet<usize> = HashSet::new();
        let mut channels = Vec::new();
        let (sender, receiver) = channel::<usize>();

        // Spawn `jobs` number of threads (but no more than the task count!).
        for thread_id in 0..thread_count {
            let path = self.environment.path().to_path_buf();
            let dry_run = self.dry_run;
            let thread_sender = sender.clone();

            let (parent_sender, thread_receiver) = channel::<String>();
            channels.push(parent_sender);

            free_threads.insert(thread_id);
            threads.push(thread::spawn(move || {
                // Prepare a new environment.
                let environment = Environment::new(path).unwrap_or_else(|e| {
                    error!("{}", e);
                    panic!();
                });

                // Open standard library functions.
                stdlib::open_lib(environment.clone());

                // Load the script.
                if let Err(e) = environment.load() {
                    error!("{}", e);
                    panic!();
                }

                thread_sender.send(thread_id).unwrap();

                // Begin executing tasks!
                while let Ok(name) = thread_receiver.recv() {
                    // Lookup the task to run.
                    let task = {
                        // Lookup the task to run.
                        if let Some(task) = environment.get_task(&name) {
                            task as Rc<Task>
                        }

                        // Find a rule that matches the task name.
                        else if let Some(rule) = environment.rules().iter().find(|rule| rule.matches(&name)) {
                            Rc::new(rule.create_task(name).unwrap()) as Rc<Task>
                        }

                        // No matching task.
                        else {
                            panic!("no matching task or rule for '{}'", name);
                        }
                    };

                    // Check for dry run.
                    if !dry_run {
                        if let Err(e) = task.run() {
                            error!("{}", e);
                            panic!();
                        }
                    } else {
                        info!("would run task '{}'", task.name());
                    }

                    thread_sender.send(thread_id).unwrap_or(());
                }
            }))
        }

        drop(sender);

        // Keep track of tasks completed and tasks in progress.
        let mut completed_tasks: HashSet<String> = HashSet::new();
        let mut current_tasks: HashMap<usize, String> = HashMap::new();

        while !schedule.is_empty() {
            // Wait for a thread to request a task.
            let thread_id = receiver.recv().unwrap();
            free_threads.insert(thread_id);
            trace!("thread {} is idle", thread_id);

            // If the thread was previously running a task, mark it as completed.
            if let Some(task) = current_tasks.remove(&thread_id) {
                trace!("task {} completed", task);
                completed_tasks.insert(task);
            }

            // On the last thread, the schedule will be empty.
            if schedule.is_empty() {
                break;
            }

            // Attempt to schedule more tasks to run. The most we can schedule is the number of free
            // threads, but it is limited by the number of tasks that have their dependencies already
            // finished.
            for _ in 0..free_threads.len() {
                // Check the next task in the queue.
                if schedule.len() > 0 && schedule.front().unwrap().dependencies().iter().all(|d| {
                    completed_tasks.contains(d) || schedule.iter().find(|a| a.name() == d).is_none()
                }) {
                    // Pop the available task from the queue.
                    let task = schedule.pop_front().unwrap();

                    println!("[{}/{}] {}", task_count - schedule.len(), task_count, task.name());

                    // Pick a free thread to run the task in.
                    if let Some(thread_id) = free_threads.iter().next().map(|t| *t) {
                        trace!("scheduling task '{}' on thread {}", task.name(), thread_id);

                        current_tasks.insert(thread_id, task.name().to_string());
                        free_threads.remove(&thread_id);
                        channels[thread_id].send(task.name().to_string()).unwrap();
                    } else {
                        // We can schedule now, but there aren't any free threads. 😢
                        break;
                    }
                } else {
                    // We can't run the next task, so we're done scheduling for now until another
                    // thread finishes.
                    break;
                }
            }
        }

        trace!("all tasks scheduled");

        // Close the threads and channels.
        drop(channels);
        drop(receiver);

        // Wait for the threads to finish.
        threads.into_iter().map(|t| t.join()).collect::<Vec<_>>();

        Ok(())
    }

    fn resolve_task<S: AsRef<str>>(&mut self, name: S) -> Result<(), Box<Error>> {
        if !self.graph.contains(&name) {
            // Lookup the task to run.
            if let Some(task) = self.environment.get_task(&name) {
                debug!("task '{}' matches named task", name.as_ref());
                self.graph.insert(task.clone());
            }

            // Find a rule that matches the task name.
            else if let Some(rule) = self.environment.rules().iter().find(|rule| rule.matches(&name)) {
                debug!("task '{}' matches rule '{}'", name.as_ref(), rule.pattern);
                // Create a task for the rule and insert it in the graph.
                self.graph.insert(Rc::new(rule.create_task(name.as_ref()).unwrap()));
            }

            // No matching task.
            else {
                return Err(format!("no matching task or rule for '{}'", name.as_ref()).into());
            }
        }

        for dependency in self.graph.get(name).unwrap().dependencies() {
            if !self.graph.contains(dependency) {
                try!(self.resolve_task(dependency));
            }
        }

        Ok(())
    }
}
<commit_msg>Check errors and improve scheduling algorithm<commit_after>use graph::Graph;
use num_cpus;
use script::Environment;
use script::task::Task;
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::rc::Rc;
use std::sync::mpsc::channel;
use std::thread;
use stdlib;

/// A task runner object that holds the state for defined tasks, dependencies, and the scripting
/// runtime.
pub struct Runner {
    /// The script environment.
    environment: Environment,

    /// The current DAG for tasks.
    graph: Graph,

    /// Indicates if actually running tasks should be skipped.
    dry_run: bool,

    /// Indicates if up-to-date tasks should be run anyway.
    always_run: bool,

    /// The number of threads to use.
    jobs: usize,
}

impl Runner {
    /// Creates a new runner instance.
    pub fn new(environment: Environment) -> Runner {
        // By default, set the number of jobs to be one less than the number of available CPU cores.
        let jobs = cmp::max(1, num_cpus::get() - 1);

        Runner {
            environment: environment,
            graph: Graph::new(),
            dry_run: false,
            always_run: false,
            jobs: jobs as usize,
        }
    }

    /// Sets the runner to "dry run" mode.
    ///
    /// When in "dry run" mode, running tasks will operate as normal, except that no task's actions
    /// will be actually run.
    pub fn dry_run(&mut self) {
        self.dry_run = true;
    }

    /// Run all tasks even if they are up-to-date.
    pub fn always_run(&mut self) {
        self.always_run = true;
    }

    /// Sets the number of threads to use to run tasks.
    pub fn jobs(&mut self, jobs: usize) {
        self.jobs = jobs;
    }

    /// Run the default task.
    pub fn run_default(&mut self) -> Result<(), Box<Error>> {
        if let Some(ref name) = self.environment.default_task() {
            let tasks = vec![name];
            self.run(&tasks)
        } else {
            Err("no default task defined".into())
        }
    }

    /// Runs the specified list of tasks.
    ///
    /// Tasks are run in parallel when possible during execution. The maximum number of parallel
    /// jobs can be set with the `jobs()` method.
    pub fn run<S: AsRef<str>>(&mut self, tasks: &[S]) -> Result<(), Box<Error>> {
        // Resolve all tasks given.
        for task in tasks {
            try!(self.resolve_task(task));
        }

        // Determine the schedule of tasks to execute.
        let mut schedule = try!(self.graph.solve(!self.always_run));
        let task_count = schedule.len();
        let thread_count = cmp::min(self.jobs, task_count);

        debug!("running {} task(s) across {} thread(s)", task_count, thread_count);

        // Spawn one thread for each job.
        let mut threads = Vec::new();
        let mut free_threads: HashSet<usize> = HashSet::new();
        let mut channels = Vec::new();
        let (sender, receiver) = channel::<usize>();

        // Spawn `jobs` number of threads (but no more than the task count!).
        for thread_id in 0..thread_count {
            let path = self.environment.path().to_path_buf();
            let dry_run = self.dry_run;
            let thread_sender = sender.clone();

            let (parent_sender, thread_receiver) = channel::<(String, usize)>();
            channels.push(parent_sender);

            free_threads.insert(thread_id);
            threads.push(thread::spawn(move || {
                // Prepare a new environment.
                let environment = Environment::new(path).unwrap_or_else(|e| {
                    error!("{}", e);
                    panic!();
                });

                // Open standard library functions.
                stdlib::open_lib(environment.clone());

                // Load the script.
                if let Err(e) = environment.load() {
                    error!("{}", e);
                    panic!();
                }

                if thread_sender.send(thread_id).is_err() {
                    trace!("thread {} failed to send channel", thread_id);
                }

                // Begin executing tasks!
                while let Ok((name, task_id)) = thread_receiver.recv() {
                    println!("[{}/{}] {}", task_id, task_count, name);

                    // Lookup the task to run.
                    let task = {
                        // Lookup the task to run.
                        if let Some(task) = environment.get_task(&name) {
                            task as Rc<Task>
                        }

                        // Find a rule that matches the task name.
                        else if let Some(rule) = environment.rules().iter().find(|rule| rule.matches(&name)) {
                            Rc::new(rule.create_task(name).unwrap()) as Rc<Task>
                        }

                        // No matching task.
                        else {
                            panic!("no matching task or rule for '{}'", name);
                        }
                    };

                    // Check for dry run.
                    if !dry_run {
                        if let Err(e) = task.run() {
                            error!("{}", e);
                            panic!();
                        }
                    } else {
                        info!("would run task '{}'", task.name());
                    }

                    if thread_sender.send(thread_id).is_err() {
                        trace!("thread {} failed to send channel", thread_id);
                        break;
                    }
                }
            }))
        }

        drop(sender);

        // Keep track of tasks completed and tasks in progress.
        let mut completed_tasks: HashSet<String> = HashSet::new();
        let mut current_tasks: HashMap<usize, String> = HashMap::new();
        let all_tasks: HashSet<String> = schedule.iter().map(|s| s.name().to_string()).collect();

        while !schedule.is_empty() {
            // Wait for a thread to request a task.
            let thread_id = receiver.recv().unwrap();
            free_threads.insert(thread_id);
            trace!("thread {} is idle", thread_id);

            // If the thread was previously running a task, mark it as completed.
            if let Some(task) = current_tasks.remove(&thread_id) {
                trace!("task {} completed", task);
                completed_tasks.insert(task);
            }

            // Attempt to schedule more tasks to run. The most we can schedule is the number of free
            // threads, but it is limited by the number of tasks that have their dependencies already
            // finished.
            'schedule: for _ in 0..free_threads.len() {
                // If the schedule is empty, we are done.
                if schedule.is_empty() {
                    break;
                }

                // Check the next task in the queue. If any of its dependencies have not yet been
                // completed, we cannot schedule it yet.
                for dependency in schedule.front().unwrap().dependencies() {
                    // Check that the dependency needs scheduled at all (some are already satisfied),
                    // and that it hasn't already finished.
                    if all_tasks.contains(dependency) && !completed_tasks.contains(dependency) {
                        // We can't run the next task, so we're done scheduling for now until another
                        // thread finishes.
                        break 'schedule;
                    }
                }

                // Pop the available task from the queue.
                let task = schedule.pop_front().unwrap();

                // Pick a free thread to run the task in.
                if let Some(thread_id) = free_threads.iter().next().map(|t| *t) {
                    trace!("scheduling task '{}' on thread {}", task.name(), thread_id);
                    let data = (task.name().to_string(), task_count - schedule.len());

                    // Send the task name.
                    if channels[thread_id].send(data).is_ok() {
                        current_tasks.insert(thread_id, task.name().to_string());
                        free_threads.remove(&thread_id);
                    } else {
                        trace!("failed to send channel to thread {}", thread_id);
                    }
                } else {
                    // We can schedule now, but there aren't any free threads. 😢
                    break;
                }
            }
        }

        // Close the input and wait for any remaining threads to finish.
        drop(channels);
        for (thread_id, thread) in threads.into_iter().enumerate() {
            if let Err(e) = thread.join() {
                trace!("thread {} closed with panic: {:?}", thread_id, e);
            }
        }

        Ok(())
    }

    fn resolve_task<S: AsRef<str>>(&mut self, name: S) -> Result<(), Box<Error>> {
        if !self.graph.contains(&name) {
            // Lookup the task to run.
            if let Some(task) = self.environment.get_task(&name) {
                debug!("task '{}' matches named task", name.as_ref());
                self.graph.insert(task.clone());
            }

            // Find a rule that matches the task name.
            else if let Some(rule) = self.environment.rules().iter().find(|rule| rule.matches(&name)) {
                debug!("task '{}' matches rule '{}'", name.as_ref(), rule.pattern);
                // Create a task for the rule and insert it in the graph.
                self.graph.insert(Rc::new(rule.create_task(name.as_ref()).unwrap()));
            }

            // No matching task.
            else {
                return Err(format!("no matching task or rule for '{}'", name.as_ref()).into());
            }
        }

        for dependency in self.graph.get(name).unwrap().dependencies() {
            if !self.graph.contains(dependency) {
                try!(self.resolve_task(dependency));
            }
        }

        Ok(())
    }
}
<|endoftext|>