<commit_before>use chrono::UTC;
use chrono::datetime::DateTime;

use params::*;

use rs_es::Client;
use rs_es::query::{Filter, Query};
use rs_es::units::{JsonVal, DurationUnit};
use rs_es::units::Duration as ESDuration;
use rs_es::operations::search::{Sort, SortField, Order};
use rs_es::operations::index::IndexResult;
use rs_es::operations::bulk::Action;
use rs_es::error::EsError;

extern crate searchspot;
use searchspot::terms::VectorOfTerms;

#[derive(RustcEncodable, RustcDecodable, Debug)]
pub struct Talent {
  pub id:                 u32,
  pub accepted:           bool,
  pub work_roles:         Vec<String>,
  pub work_languages:     Vec<String>,
  pub work_experience:    String,
  pub work_locations:     Vec<String>,
  pub work_authorization: String,
  pub company_ids:        Vec<u32>,
  pub batch_starts_at:    i64,
  pub batch_ends_at:      i64,
  pub added_to_batch_at:  i64,
  pub weight:             i32,
  pub blocked_companies:  Vec<u32>
}

impl Talent {
  /// Populate the ElasticSearch index with `self`.
  pub fn index(&self, mut es: &mut Client, index: &str) -> Result<IndexResult, EsError> {
    es.index(index, "talent")
      .with_doc(&self)
      .with_id(&*format!("{}", self.id))
      .send()
  }

  #[allow(dead_code)]
  pub fn delete_all(mut es: &mut Client, index: &str) {
    let mut scan = match es.search_query()
                           .with_indexes(&[&index])
                           .with_query(&Query::build_match_all().build())
                           .scan(ESDuration::new(1, DurationUnit::Minute)) {
                              Ok(scan) => scan,
                              Err(_)   => return
                           };

    loop {
      let     page = scan.scroll(&mut es).unwrap();
      let mut hits = page.hits.hits;

      if hits.len() == 0 {
        break;
      }

      let actions = hits.drain(..)
                        .map(|hit| {
                           Action::delete(hit.id).with_index(&*index)
                                                 .with_doc_type(hit.doc_type)
                        })
                        .collect::<Vec<Action>>();
      es.bulk(&actions)
        .send()
        .unwrap();
    }

    scan.close(&mut es)
        .unwrap();
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  pub fn search(mut es: &mut Client, default_index: &str, params: &Map) -> Vec<u32> {
    let epoch = match params.find(&["epoch"]) {
      Some(&Value::I64(epoch)) => epoch,
      _ => DateTime::timestamp(&UTC::now())
    };

    let index: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => vec![default_index]
    };

    let result = es.search_query()
                   .with_indexes(&*index)
                   .with_query(&Talent::search_filters(params, epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .send();

    match result {
      Ok(result) => {
        let mut results = result.hits.hits.into_iter()
                                          .map(|hit| hit.source.unwrap()["id"]
                                                               .as_u64()
                                                               .unwrap() as u32)
                                          .collect::<Vec<u32>>();
        results.dedup();
        results
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: i64, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_starts_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build(),
                                      Filter::build_range("batch_ends_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("epoch_second")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// `VectorOfTerms` are ORred, while `Filter`s are ANDed.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_languages`, if provided, must not be empty.
  fn search_filters(params: &Map, epoch: i64) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_languages", &vec_from_params!(params, "work_languages")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &string_vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &string_vec_from_params!(params, "work_authorization")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Return a `Sort` that makes values be sorted for given fields, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("batch_starts_at",   Some(Order::Desc)).build(),
        SortField::new("weight",            Some(Order::Desc)).build(),
        SortField::new("added_to_batch_at", Some(Order::Desc)).build()
      ])
  }
}

#[cfg(test)]
mod tests {
  use chrono::*;

  use rs_es::Client;

  use params::*;

  extern crate searchspot;
  use searchspot::config::*;

  use resources::user::Talent;

  use std::time::Duration as TimeDuration;
  use std::thread::sleep;

  const CONFIG_FILE: &'static str = "examples/tests.toml";

  lazy_static! {
    static ref config: Config = Config::from_file(CONFIG_FILE.to_owned());
  }

  pub fn make_client() -> Client {
    Client::new(&*config.es.host, config.es.port)
  }

  macro_rules! epoch_from_year {
    ($year:expr) => {
      UTC.datetime_from_str(&format!("{}-01-01 12:00:00", $year),
        "%Y-%m-%d %H:%M:%S").unwrap().timestamp()
    }
  }

  pub fn populate_es(mut client: &mut Client) {
    vec![
      Talent {
        id:                 1,
        accepted:           true,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             -5,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 2,
        accepted:           true,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 3,
        accepted:           false,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2007"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 4,
        accepted:           true,
        work_roles:         vec!["Fullstack".to_owned(), "DevOps".to_owned()],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization:  "yes".to_owned(),
        company_ids:        vec![6],
        batch_starts_at:    epoch_from_year!("2008"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             0,
        blocked_companies:  vec![]
      }
    ].iter()
     .all(|user| user.index(&mut client, &config.es.index)
                     .is_ok());

    sleep(TimeDuration::from_millis(1000));
  }

  #[test]
  fn test_search() {
    let mut client = make_client();
    Talent::delete_all(&mut client, &config.es.index);
    populate_es(&mut client);

    // no parameters are given
    {
      let results = Talent::search(&mut client, &*config.es.index, &Map::new());
      assert_eq!(vec![4, 2, 1], results);
    }

    // a non existing index is given
    {
      let mut map = Map::new();
      map.assign("index", Value::String("lololol".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // a date that doesn't match given indexes is given
    {
      let mut map = Map::new();
      map.assign("epoch", Value::I64(epoch_from_year!("2040"))).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // TODO: map field type properly (see benashford/rs-es#11)
    // // filtering for valid work roles
    // // {
    // //   let mut map = Map::new();
    // //   map.assign("work_roles[]", Value::String("Fullstack".to_owned())).unwrap();
    // //
    // //   let results = Talent::search(&mut client, &*config.es.index, &map);
    // //   assert_eq!(vec![4], results);
    // // }

    // filtering for given company_id
    {
      let mut map = Map::new();
      map.assign("company_id", Value::String("6".into())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![2, 1], results);
    }
  }
}
<commit_msg>resources/user: handle dates as String and disable pagination<commit_after>use chrono::UTC;

use params::*;

use rs_es::Client;
use rs_es::query::{Filter, Query};
use rs_es::units::{JsonVal, DurationUnit};
use rs_es::units::Duration as ESDuration;
use rs_es::operations::search::{Sort, SortField, Order};
use rs_es::operations::index::IndexResult;
use rs_es::operations::bulk::Action;
use rs_es::error::EsError;

extern crate searchspot;
use searchspot::terms::VectorOfTerms;

#[derive(RustcEncodable, RustcDecodable, Debug)]
pub struct Talent {
  pub id:                 u32,
  pub accepted:           bool,
  pub work_roles:         Vec<String>,
  pub work_languages:     Vec<String>,
  pub work_experience:    String,
  pub work_locations:     Vec<String>,
  pub work_authorization: String,
  pub company_ids:        Vec<u32>,
  pub batch_starts_at:    String,
  pub batch_ends_at:      String,
  pub added_to_batch_at:  String,
  pub weight:             i32,
  pub blocked_companies:  Vec<u32>
}

impl Talent {
  /// Populate the ElasticSearch index with `self`.
  pub fn index(&self, mut es: &mut Client, index: &str) -> Result<IndexResult, EsError> {
    es.index(index, "talent")
      .with_doc(&self)
      .with_id(&*format!("{}", self.id))
      .send()
  }

  #[allow(dead_code)]
  pub fn delete_all(mut es: &mut Client, index: &str) {
    let mut scan = match es.search_query()
                           .with_indexes(&[&index])
                           .with_query(&Query::build_match_all().build())
                           .scan(ESDuration::new(1, DurationUnit::Minute)) {
                              Ok(scan) => scan,
                              Err(_)   => return
                           };

    loop {
      let     page = scan.scroll(&mut es).unwrap();
      let mut hits = page.hits.hits;

      if hits.len() == 0 {
        break;
      }

      let actions = hits.drain(..)
                        .map(|hit| {
                           Action::delete(hit.id).with_index(&*index)
                                                 .with_doc_type(hit.doc_type)
                        })
                        .collect::<Vec<Action>>();
      es.bulk(&actions)
        .send()
        .unwrap();
    }

    scan.close(&mut es)
        .unwrap();
  }

  /// Query ElasticSearch on given `indexes` and `params` and return the IDs of
  /// the found talents.
  pub fn search(mut es: &mut Client, default_index: &str, params: &Map) -> Vec<u32> {
    let now   = UTC::now().to_rfc3339();
    let epoch = match params.find(&["epoch"]) {
      Some(epoch) => String::from_value(&epoch).unwrap_or(now),
      _           => now
    };

    let index: Vec<&str> = match params.find(&["index"]) {
      Some(&Value::String(ref index)) => vec![&index[..]],
      _ => vec![default_index]
    };

    let result = es.search_query()
                   .with_indexes(&*index)
                   .with_query(&Talent::search_filters(params, &*epoch))
                   .with_sort(&Talent::sorting_criteria())
                   .with_size(1000) // TODO
                   .send();

    match result {
      Ok(result) => {
        let mut results = result.hits.hits.into_iter()
                                          .map(|hit| hit.source.unwrap()["id"]
                                                               .as_u64()
                                                               .unwrap() as u32)
                                          .collect::<Vec<u32>>();
        results.dedup();
        results
      },
      Err(err) => {
        println!("{:?}", err);
        vec![]
      }
    }
  }

  /// Return a `Vec<Filter>` with visibility criteria for the talents.
  /// The `epoch` must be given as `I64` (UNIX time in seconds) and is
  /// the range in which batches are searched.
  /// If `presented_talents` is provided, talents who match the IDs
  /// contained there skip the standard visibility criteria.
  ///
  /// Basically, the talents must be accepted into the platform and must be
  /// inside a living batch to match the visibility criteria.
  fn visibility_filters(epoch: &str, presented_talents: Vec<i32>) -> Vec<Filter> {
    let visibility_rules = Filter::build_bool()
                                  .with_must(
                                    vec![
                                      Filter::build_term("accepted", true)
                                             .build(),
                                      Filter::build_range("batch_starts_at")
                                             .with_lte(JsonVal::from(epoch))
                                             .with_format("strict_date_optional_time")
                                             .build(),
                                      Filter::build_range("batch_ends_at")
                                             .with_gte(JsonVal::from(epoch))
                                             .with_format("strict_date_optional_time")
                                             .build()
                                    ])
                                  .build();

    if presented_talents.len() > 0 { // preferred over !_.is_empty()
      let presented_talents_filters = Filter::build_bool()
                                             .with_must(
                                               vec![
                                                 <Filter as VectorOfTerms<i32>>::build_terms(
                                                   "ids", &presented_talents)
                                               ].into_iter()
                                                .flat_map(|x| x)
                                                .collect::<Vec<Filter>>())
                                             .build();
      vec![
        Filter::build_bool()
               .with_should(vec![visibility_rules, presented_talents_filters])
               .build()
      ]
    }
    else {
      vec![visibility_rules]
    }
  }

  /// Given parameters inside the query string mapped inside a `Map`,
  /// and the `epoch` (defined as UNIX time in seconds) for batches,
  /// return a `Query` for ElasticSearch.
  ///
  /// `VectorOfTerms` are ORred, while `Filter`s are ANDed.
  /// I.e.: given ["Fullstack", "DevOps"] as `work_roles`, found talents
  /// will present at least one of these roles), but both `work_roles`
  /// and `work_languages`, if provided, must not be empty.
  fn search_filters(params: &Map, epoch: &str) -> Query {
    let company_id = i32_vec_from_params!(params, "company_id");

    Query::build_filtered(Filter::build_bool()
                                 .with_must(
                                   vec![
                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_roles", &vec_from_params!(params, "work_roles")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_languages", &vec_from_params!(params, "work_languages")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_experience", &string_vec_from_params!(params, "work_experience")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                      "work_authorization", &string_vec_from_params!(params, "work_authorization")),

                                     <Filter as VectorOfTerms<String>>::build_terms(
                                       "work_locations", &vec_from_params!(params, "work_locations")),

                                     Talent::visibility_filters(epoch,
                                       i32_vec_from_params!(params, "presented_talents"))
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .with_must_not(
                                   vec![
                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "company_ids", &company_id),

                                     <Filter as VectorOfTerms<i32>>::build_terms(
                                       "blocked_companies", &company_id)
                                   ].into_iter()
                                    .flat_map(|x| x)
                                    .collect::<Vec<Filter>>())
                                 .build())
          .build()
  }

  /// Return a `Sort` that makes values be sorted for given fields, descendently.
  fn sorting_criteria() -> Sort {
    Sort::new(
      vec![
        SortField::new("batch_starts_at",   Some(Order::Desc)).build(),
        SortField::new("weight",            Some(Order::Desc)).build(),
        SortField::new("added_to_batch_at", Some(Order::Desc)).build()
      ])
  }
}

#[cfg(test)]
mod tests {
  use chrono::*;

  use rs_es::Client;

  use params::*;

  extern crate searchspot;
  use searchspot::config::*;

  use resources::user::Talent;

  use std::time::Duration as TimeDuration;
  use std::thread::sleep;

  const CONFIG_FILE: &'static str = "examples/tests.toml";

  lazy_static! {
    static ref config: Config = Config::from_file(CONFIG_FILE.to_owned());
  }

  pub fn make_client() -> Client {
    Client::new(&*config.es.host, config.es.port)
  }

  macro_rules! epoch_from_year {
    ($year:expr) => {
      UTC.datetime_from_str(&format!("{}-01-01 12:00:00", $year),
        "%Y-%m-%d %H:%M:%S").unwrap().to_rfc3339()
    }
  }

  pub fn populate_es(mut client: &mut Client) {
    vec![
      Talent {
        id:                 1,
        accepted:           true,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             -5,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 2,
        accepted:           true,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2006"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2006"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 3,
        accepted:           false,
        work_roles:         vec![],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization: "yes".to_owned(),
        company_ids:        vec![],
        batch_starts_at:    epoch_from_year!("2007"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             6,
        blocked_companies:  vec![]
      },

      Talent {
        id:                 4,
        accepted:           true,
        work_roles:         vec!["Fullstack".to_owned(), "DevOps".to_owned()],
        work_languages:     vec![],
        work_experience:    "1..2".to_owned(),
        work_locations:     vec!["Berlin".to_owned()],
        work_authorization:  "yes".to_owned(),
        company_ids:        vec![6],
        batch_starts_at:    epoch_from_year!("2008"),
        batch_ends_at:      epoch_from_year!("2020"),
        added_to_batch_at:  epoch_from_year!("2011"),
        weight:             0,
        blocked_companies:  vec![]
      }
    ].iter()
     .all(|user| user.index(&mut client, &config.es.index)
                     .is_ok());

    sleep(TimeDuration::from_millis(5000));
  }

  #[test]
  fn test_search() {
    let mut client = make_client();
    Talent::delete_all(&mut client, &config.es.index);
    populate_es(&mut client);

    // no parameters are given
    {
      let results = Talent::search(&mut client, &*config.es.index, &Map::new());
      assert_eq!(vec![4, 2, 1], results);
    }

    // a non existing index is given
    {
      let mut map = Map::new();
      map.assign("index", Value::String("lololol".to_owned())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // a date that doesn't match given indexes is given
    {
      let mut map = Map::new();
      map.assign("epoch", Value::String(epoch_from_year!("2040"))).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert!(results.is_empty());
    }

    // TODO: map field type properly (see benashford/rs-es#11)
    // // filtering for valid work roles
    // // {
    // //   let mut map = Map::new();
    // //   map.assign("work_roles[]", Value::String("Fullstack".to_owned())).unwrap();
    // //
    // //   let results = Talent::search(&mut client, &*config.es.index, &map);
    // //   assert_eq!(vec![4], results);
    // // }

    // filtering for given company_id
    {
      let mut map = Map::new();
      map.assign("company_id", Value::String("6".into())).unwrap();

      let results = Talent::search(&mut client, &*config.es.index, &map);
      assert_eq!(vec![2, 1], results);
    }
  }
}
<|endoftext|>


<commit_before>//! Various source mapping utilities

use std::borrow::Cow;
use std::{fmt, io};
use std::path::PathBuf;

use pos::{ByteOffset, BytePos, ColumnIndex, LineIndex, RawOffset, RawPos, Span};

#[derive(Clone, Debug)]
pub enum FileName {
    /// A real file on disk
    Real(PathBuf),
    /// A synthetic file, eg. from the REPL
    Virtual(Cow<'static, str>),
}

impl FileName {
    pub fn real<T: Into<PathBuf>>(name: T) -> FileName {
        FileName::Real(name.into())
    }

    pub fn virtual_<T: Into<Cow<'static, str>>>(name: T) -> FileName {
        FileName::Virtual(name.into())
    }
}

impl fmt::Display for FileName {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            FileName::Real(ref path) => write!(fmt, "{}", path.display()),
            FileName::Virtual(ref name) => write!(fmt, "<{}>", name),
        }
    }
}

#[derive(Debug, Fail, PartialEq)]
pub enum LineIndexError {
    #[fail(display = "Line out of bounds - given: {:?}, max: {:?}", given, max)]
    OutOfBounds { given: LineIndex, max: LineIndex },
}

#[derive(Debug, Fail, PartialEq)]
pub enum BytePosError {
    #[fail(display = "Byte position out of bounds - given: {}, span: {}", given, span)]
    OutOfBounds { given: BytePos, span: Span },
}

#[derive(Debug, Fail, PartialEq)]
pub enum SpanError {
    #[fail(display = "Span out of bounds - given: {}, span: {}", given, span)]
    OutOfBounds { given: Span, span: Span },
}

#[derive(Debug)]
/// Some source code
pub struct FileMap {
    /// The name of the file that the source came from
    name: FileName,
    /// The complete source code
    src: String,
    /// The span of the source in the `CodeMap`
    span: Span,
    /// Offsets to the line beginnings in the source
    lines: Vec<ByteOffset>,
}

impl FileMap {
    /// Construct a new filemap, creating an index of line start locations
    pub(crate) fn new(name: FileName, src: String, start_pos: BytePos) -> FileMap {
        use std::iter;

        let span = Span::from_offset(start_pos, ByteOffset::from_str(&src));
        let lines = {
            let newline_off = ByteOffset::from_char_utf8('\n');
            let offsets = src.match_indices('\n')
                .map(|(i, _)| ByteOffset(i as RawOffset) + newline_off);

            iter::once(ByteOffset(0)).chain(offsets).collect()
        };

        FileMap {
            name,
            src,
            span,
            lines,
        }
    }

    /// Read some source code from a file, loading it into a filemap
    pub(crate) fn from_disk<P: Into<PathBuf>>(name: P, start_pos: BytePos) -> io::Result<FileMap> {
        use std::fs::File;
        use std::io::Read;

        let name = name.into();
        let mut file = File::open(&name)?;
        let mut src = String::new();
        file.read_to_string(&mut src)?;

        Ok(FileMap::new(FileName::Real(name), src, start_pos))
    }

    /// The name of the file that the source came from
    pub fn name(&self) -> &FileName {
        &self.name
    }

    /// The underlying source code
    pub fn src(&self) -> &str {
        &self.src
    }

    /// The span of the source in the `CodeMap`
    pub fn span(&self) -> Span {
        self.span
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_offset(&self, index: LineIndex) -> Result<ByteOffset, LineIndexError> {
        self.lines
            .get(index.to_usize())
            .cloned()
            .ok_or_else(|| LineIndexError::OutOfBounds {
                given: index,
                max: LineIndex(self.lines.len() as RawPos - 1),
            })
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_pos(&self, index: LineIndex) -> Result<BytePos, LineIndexError> {
        self.line_offset(index)
            .map(|offset| self.span.lo() + offset)
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_span(&self, line: LineIndex) -> Result<Span, LineIndexError> {
        let lo = self.span.lo() + self.line_offset(line)?;
        let hi = match self.line_offset(LineIndex(line.0 + 1)) {
            Ok(offset_hi) => self.span.lo() + offset_hi,
            Err(_) => self.span.hi(),
        };

        Ok(Span::new(hi, lo))
    }

    /// Returns the line and column location of `byte`
    pub fn location(&self, pos: BytePos) -> Result<(LineIndex, ColumnIndex), BytePosError> {
        let line_index = self.find_line_at_pos(pos)?;
        let line_span = self.line_span(line_index).unwrap(); // line_index should be valid!
        let line_slice = self.src_slice(line_span).unwrap(); // line_span should be valid!
        let byte_col = pos - line_span.lo();
        let column_index = ColumnIndex(line_slice[..byte_col.to_usize()].chars().count() as RawPos);

        Ok((line_index, column_index))
    }

    /// Returns the line index that the byte position points to
    pub fn find_line_at_pos(&self, pos: BytePos) -> Result<LineIndex, BytePosError> {
        let span = self.span;
        match () {
            () if pos < span.lo() => Err(BytePosError::OutOfBounds { given: pos, span }),
            () if pos > span.hi() => Err(BytePosError::OutOfBounds { given: pos, span }),
            () => match self.lines.binary_search(&(pos - span.lo())) {
                Ok(i) => Ok(LineIndex(i as RawPos)),
                Err(i) => Ok(LineIndex(i as RawPos - 1)),
            },
        }
    }

    /// Get the corresponding source string for a span
    ///
    /// Returns `Err` if the span is outside the bounds of the file
    pub fn src_slice(&self, span: Span) -> Result<&str, SpanError> {
        match () {
            () if !self.span.contains(span) => Err(SpanError::OutOfBounds {
                given: span,
                span: self.span,
            }),
            () => {
                let lo = (span.lo() - self.span.lo()).to_usize();
                let hi = (span.hi() - self.span.lo()).to_usize();

                Ok(&self.src[lo..hi])
            },
        }
    }
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;
    use {CodeMap, FileMap, FileName};

    use super::*;

    struct TestData {
        filemap: Arc<FileMap>,
        lines: &'static [&'static str],
    }

    impl TestData {
        fn new() -> TestData {
            let mut codemap = CodeMap::new();
            let lines = &["hello!\n", "howdy\n", "\n", "hi萤\n", "bloop\n"];
            let filemap = codemap.add_filemap(FileName::Virtual("test".into()), lines.concat());

            TestData { filemap, lines }
        }

        fn byte_offsets(&self) -> Vec<ByteOffset> {
            let mut offset = ByteOffset(0);
            let mut byte_offsets = Vec::new();

            for line in self.lines {
                byte_offsets.push(offset);
                offset += ByteOffset::from_str(line);
                byte_offsets.push(offset + -ByteOffset::from_char_utf8('\n'));
            }

            // bump us past the end
            byte_offsets.push(offset);

            byte_offsets
        }

        fn byte_positions(&self) -> Vec<BytePos> {
            let mut offsets = vec![BytePos::none()];
            offsets.extend(self.byte_offsets().iter().map(|&off| BytePos(1) + off));
            let out_of_bounds = *offsets.last().unwrap() + ByteOffset(1);
            offsets.push(out_of_bounds);
            offsets
        }

        fn line_indices(&self) -> Vec<LineIndex> {
            (0..self.lines.len() + 2)
                .map(|i| LineIndex(i as RawPos))
                .collect()
        }
    }

    #[test]
    fn line_offset() {
        let test_data = TestData::new();
        let offsets: Vec<_> = test_data
            .line_indices()
            .iter()
            .map(|&i| test_data.filemap.line_offset(i))
            .collect();

        assert_eq!(
            offsets,
            vec![
                Ok(ByteOffset(0)),
                Ok(ByteOffset(7)),
                Ok(ByteOffset(13)),
                Ok(ByteOffset(14)),
                Ok(ByteOffset(20)),
                Ok(ByteOffset(26)),
                Err(LineIndexError::OutOfBounds {
                    given: LineIndex(6),
                    max: LineIndex(5),
                }),
            ],
        );
    }

    #[test]
    fn line_pos() {
        let test_data = TestData::new();
        let offsets: Vec<_> = test_data
            .line_indices()
            .iter()
            .map(|&i| test_data.filemap.line_pos(i))
            .collect();

        assert_eq!(
            offsets,
            vec![
                Ok(test_data.filemap.span().lo() + ByteOffset(0)),
                Ok(test_data.filemap.span().lo() + ByteOffset(7)),
                Ok(test_data.filemap.span().lo() + ByteOffset(13)),
                Ok(test_data.filemap.span().lo() + ByteOffset(14)),
                Ok(test_data.filemap.span().lo() + ByteOffset(20)),
                Ok(test_data.filemap.span().lo() + ByteOffset(26)),
                Err(LineIndexError::OutOfBounds {
                    given: LineIndex(6),
                    max: LineIndex(5),
                }),
            ],
        );
    }

    // #[test]
    // fn line_span() {
    //     let filemap = filemap();
    //     let lo = filemap.span().lo();

    //     assert_eq!(filemap.line_pos(Li(0)), Some(lo + BOff(0)));
    //     assert_eq!(filemap.line_pos(Li(1)), Some(lo + BOff(7)));
    //     assert_eq!(filemap.line_pos(Li(2)), Some(lo + BOff(13)));
    //     assert_eq!(filemap.line_pos(Li(3)), Some(lo + BOff(14)));
    //     assert_eq!(filemap.line_pos(Li(4)), Some(lo + BOff(20)));
    //     assert_eq!(filemap.line_pos(Li(5)), Some(lo + BOff(26)));
    //     assert_eq!(filemap.line_pos(Li(6)), None);
    // }

    #[test]
    fn location() {
        let test_data = TestData::new();
        let lines: Vec<_> = test_data
            .byte_positions()
            .iter()
            .map(|&pos| test_data.filemap.location(pos))
            .collect();

        assert_eq!(
            lines,
            vec![
                Err(BytePosError::OutOfBounds {
                    given: BytePos(0),
                    span: test_data.filemap.span(),
                }),
                Ok((LineIndex(0), ColumnIndex(0))),
                Ok((LineIndex(0), ColumnIndex(6))),
                Ok((LineIndex(1), ColumnIndex(0))),
                Ok((LineIndex(1), ColumnIndex(5))),
                Ok((LineIndex(2), ColumnIndex(0))),
                Ok((LineIndex(2), ColumnIndex(0))),
                Ok((LineIndex(3), ColumnIndex(0))),
                Ok((LineIndex(3), ColumnIndex(3))),
                Ok((LineIndex(4), ColumnIndex(0))),
                Ok((LineIndex(4), ColumnIndex(5))),
                Ok((LineIndex(5), ColumnIndex(0))),
                Err(BytePosError::OutOfBounds {
                    given: BytePos(28),
                    span: test_data.filemap.span(),
                }),
            ],
        );
    }

    #[test]
    fn find_line_at_pos() {
        let test_data = TestData::new();
        let lines: Vec<_> = test_data
            .byte_positions()
            .iter()
            .map(|&pos| test_data.filemap.find_line_at_pos(pos))
            .collect();

        assert_eq!(
            lines,
            vec![
                Err(BytePosError::OutOfBounds {
                    given: BytePos(0),
                    span: test_data.filemap.span(),
                }),
                Ok(LineIndex(0)),
                Ok(LineIndex(0)),
                Ok(LineIndex(1)),
                Ok(LineIndex(1)),
                Ok(LineIndex(2)),
                Ok(LineIndex(2)),
                Ok(LineIndex(3)),
                Ok(LineIndex(3)),
                Ok(LineIndex(4)),
                Ok(LineIndex(4)),
                Ok(LineIndex(5)),
                Err(BytePosError::OutOfBounds {
                    given: BytePos(28),
                    span: test_data.filemap.span(),
                }),
            ],
        );
    }
}
<commit_msg>Check character boundaries on find_line_at_pos<commit_after>//! Various source mapping utilities

use std::borrow::Cow;
use std::{fmt, io};
use std::path::PathBuf;

use pos::{ByteOffset, BytePos, ColumnIndex, LineIndex, RawOffset, RawPos, Span};

#[derive(Clone, Debug)]
pub enum FileName {
    /// A real file on disk
    Real(PathBuf),
    /// A synthetic file, eg. from the REPL
    Virtual(Cow<'static, str>),
}

impl FileName {
    pub fn real<T: Into<PathBuf>>(name: T) -> FileName {
        FileName::Real(name.into())
    }

    pub fn virtual_<T: Into<Cow<'static, str>>>(name: T) -> FileName {
        FileName::Virtual(name.into())
    }
}

impl fmt::Display for FileName {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            FileName::Real(ref path) => write!(fmt, "{}", path.display()),
            FileName::Virtual(ref name) => write!(fmt, "<{}>", name),
        }
    }
}

#[derive(Debug, Fail, PartialEq)]
pub enum LineIndexError {
    #[fail(display = "Line out of bounds - given: {:?}, max: {:?}", given, max)]
    OutOfBounds { given: LineIndex, max: LineIndex },
}

#[derive(Debug, Fail, PartialEq)]
pub enum BytePosError {
    #[fail(display = "Byte position out of bounds - given: {}, span: {}", given, span)]
    OutOfBounds { given: BytePos, span: Span },
    #[fail(display = "Byte position points within a character boundary - given: {}", given)]
    InvalidCharBoundary { given: BytePos },
}

#[derive(Debug, Fail, PartialEq)]
pub enum SpanError {
    #[fail(display = "Span out of bounds - given: {}, span: {}", given, span)]
    OutOfBounds { given: Span, span: Span },
}

#[derive(Debug)]
/// Some source code
pub struct FileMap {
    /// The name of the file that the source came from
    name: FileName,
    /// The complete source code
    src: String,
    /// The span of the source in the `CodeMap`
    span: Span,
    /// Offsets to the line beginnings in the source
    lines: Vec<ByteOffset>,
}

impl FileMap {
    /// Construct a new filemap, creating an index of line start locations
    pub(crate) fn new(name: FileName, src: String, start_pos: BytePos) -> FileMap {
        use std::iter;

        let span = Span::from_offset(start_pos, ByteOffset::from_str(&src));
        let lines = {
            let newline_off = ByteOffset::from_char_utf8('\n');
            let offsets = src.match_indices('\n')
                .map(|(i, _)| ByteOffset(i as RawOffset) + newline_off);

            iter::once(ByteOffset(0)).chain(offsets).collect()
        };

        FileMap {
            name,
            src,
            span,
            lines,
        }
    }

    /// Read some source code from a file, loading it into a filemap
    pub(crate) fn from_disk<P: Into<PathBuf>>(name: P, start_pos: BytePos) -> io::Result<FileMap> {
        use std::fs::File;
        use std::io::Read;

        let name = name.into();
        let mut file = File::open(&name)?;
        let mut src = String::new();
        file.read_to_string(&mut src)?;

        Ok(FileMap::new(FileName::Real(name), src, start_pos))
    }

    /// The name of the file that the source came from
    pub fn name(&self) -> &FileName {
        &self.name
    }

    /// The underlying source code
    pub fn src(&self) -> &str {
        &self.src
    }

    /// The span of the source in the `CodeMap`
    pub fn span(&self) -> Span {
        self.span
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_offset(&self, index: LineIndex) -> Result<ByteOffset, LineIndexError> {
        self.lines
            .get(index.to_usize())
            .cloned()
            .ok_or_else(|| LineIndexError::OutOfBounds {
                given: index,
                max: LineIndex(self.lines.len() as RawPos - 1),
            })
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_pos(&self, index: LineIndex) -> Result<BytePos, LineIndexError> {
        self.line_offset(index)
            .map(|offset| self.span.lo() + offset)
    }

    /// Returns the byte offset to the start of `line`
    pub fn line_span(&self, line: LineIndex) -> Result<Span, LineIndexError> {
        let lo = self.span.lo() + self.line_offset(line)?;
        let hi = match self.line_offset(LineIndex(line.0 + 1)) {
            Ok(offset_hi) => self.span.lo() + offset_hi,
            Err(_) => self.span.hi(),
        };

        Ok(Span::new(hi, lo))
    }

    /// Returns the line and column location of `byte`
    pub fn location(&self, pos: BytePos) -> Result<(LineIndex, ColumnIndex), BytePosError> {
        let line_index = self.find_line_at_pos(pos)?;
        let line_span = self.line_span(line_index).unwrap(); // line_index should be valid!
        let line_slice = self.src_slice(line_span).unwrap(); // line_span should be valid!
        let byte_col = pos - line_span.lo();
        let column_index = ColumnIndex(line_slice[..byte_col.to_usize()].chars().count() as RawPos);

        Ok((line_index, column_index))
    }

    /// Returns the line index that the byte position points to
    pub fn find_line_at_pos(&self, pos: BytePos) -> Result<LineIndex, BytePosError> {
        if pos < self.span.lo() {
            Err(BytePosError::OutOfBounds {
                given: pos,
                span: self.span,
            })
        } else if pos > self.span.hi() {
            Err(BytePosError::OutOfBounds {
                given: pos,
                span: self.span,
            })
        } else {
            let offset = pos - self.span.lo();

            if self.src.is_char_boundary(offset.to_usize()) {
                match self.lines.binary_search(&offset) {
                    Ok(i) => Ok(LineIndex(i as RawPos)),
                    Err(i) => Ok(LineIndex(i as RawPos - 1)),
                }
            } else {
                Err(BytePosError::InvalidCharBoundary {
                    given: self.span.lo(),
                })
            }
        }
    }

    /// Get the corresponding source string for a span
    ///
    /// Returns `Err` if the span is outside the bounds of the file
    pub fn src_slice(&self, span: Span) -> Result<&str, SpanError> {
        if self.span.contains(span) {
            let lo = (span.lo() - self.span.lo()).to_usize();
            let hi = (span.hi() - self.span.lo()).to_usize();

            // TODO: check char boundaries
            Ok(&self.src[lo..hi])
        } else {
            Err(SpanError::OutOfBounds {
                given: span,
                span: self.span,
            })
        }
    }
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;
    use {CodeMap, FileMap, FileName};

    use super::*;

    struct TestData {
        filemap: Arc<FileMap>,
        lines: &'static [&'static str],
    }

    impl TestData {
        fn new() -> TestData {
            let mut codemap = CodeMap::new();
            let lines = &["hello!\n", "howdy\n", "\n", "hi萤\n", "bloop\n"];
            let filemap = codemap.add_filemap(FileName::Virtual("test".into()), lines.concat());

            TestData { filemap, lines }
        }

        fn byte_offsets(&self) -> Vec<ByteOffset> {
            let mut offset = ByteOffset(0);
            let mut byte_offsets = Vec::new();

            for line in self.lines {
                byte_offsets.push(offset);
                offset += ByteOffset::from_str(line);
                byte_offsets.push(offset + -ByteOffset::from_char_utf8('\n'));
            }

            // bump us past the end
            byte_offsets.push(offset);

            byte_offsets
        }

        fn byte_positions(&self) -> Vec<BytePos> {
            let mut offsets = vec![BytePos::none()];
            offsets.extend(self.byte_offsets().iter().map(|&off| BytePos(1) + off));
            let out_of_bounds = *offsets.last().unwrap() + ByteOffset(1);
            offsets.push(out_of_bounds);
            offsets
        }

        fn line_indices(&self) -> Vec<LineIndex> {
            (0..self.lines.len() + 2)
                .map(|i| LineIndex(i as RawPos))
                .collect()
        }
    }

    #[test]
    fn line_offset() {
        let test_data = TestData::new();
        let offsets: Vec<_> = test_data
            .line_indices()
            .iter()
            .map(|&i| test_data.filemap.line_offset(i))
            .collect();

        assert_eq!(
            offsets,
            vec![
                Ok(ByteOffset(0)),
                Ok(ByteOffset(7)),
                Ok(ByteOffset(13)),
                Ok(ByteOffset(14)),
                Ok(ByteOffset(20)),
                Ok(ByteOffset(26)),
                Err(LineIndexError::OutOfBounds {
                    given: LineIndex(6),
                    max: LineIndex(5),
                }),
            ],
        );
    }

    #[test]
    fn line_pos() {
        let test_data = TestData::new();
        let offsets: Vec<_> = test_data
            .line_indices()
            .iter()
            .map(|&i| test_data.filemap.line_pos(i))
            .collect();

        assert_eq!(
            offsets,
            vec![
                Ok(test_data.filemap.span().lo() + ByteOffset(0)),
                Ok(test_data.filemap.span().lo() + ByteOffset(7)),
                Ok(test_data.filemap.span().lo() + ByteOffset(13)),
                Ok(test_data.filemap.span().lo() + ByteOffset(14)),
                Ok(test_data.filemap.span().lo() + ByteOffset(20)),
                Ok(test_data.filemap.span().lo() + ByteOffset(26)),
                Err(LineIndexError::OutOfBounds {
                    given: LineIndex(6),
                    max: LineIndex(5),
                }),
            ],
        );
    }

    // #[test]
    // fn line_span() {
    //     let filemap = filemap();
    //     let lo = filemap.span().lo();

    //     assert_eq!(filemap.line_pos(Li(0)), Some(lo + BOff(0)));
    //     assert_eq!(filemap.line_pos(Li(1)), Some(lo + BOff(7)));
    //     assert_eq!(filemap.line_pos(Li(2)), Some(lo + BOff(13)));
    //     assert_eq!(filemap.line_pos(Li(3)), Some(lo + BOff(14)));
    //     assert_eq!(filemap.line_pos(Li(4)), Some(lo + BOff(20)));
    //     assert_eq!(filemap.line_pos(Li(5)), Some(lo + BOff(26)));
    //     assert_eq!(filemap.line_pos(Li(6)), None);
    // }

    #[test]
    fn location() {
        let test_data = TestData::new();
        let lines: Vec<_> = test_data
            .byte_positions()
            .iter()
            .map(|&pos| test_data.filemap.location(pos))
            .collect();

        assert_eq!(
            lines,
            vec![
                Err(BytePosError::OutOfBounds {
                    given: BytePos(0),
                    span: test_data.filemap.span(),
                }),
                Ok((LineIndex(0), ColumnIndex(0))),
                Ok((LineIndex(0), ColumnIndex(6))),
                Ok((LineIndex(1), ColumnIndex(0))),
                Ok((LineIndex(1), ColumnIndex(5))),
                Ok((LineIndex(2), ColumnIndex(0))),
                Ok((LineIndex(2), ColumnIndex(0))),
                Ok((LineIndex(3), ColumnIndex(0))),
                Ok((LineIndex(3), ColumnIndex(3))),
                Ok((LineIndex(4), ColumnIndex(0))),
                Ok((LineIndex(4), ColumnIndex(5))),
                Ok((LineIndex(5), ColumnIndex(0))),
                Err(BytePosError::OutOfBounds {
                    given: BytePos(28),
                    span: test_data.filemap.span(),
                }),
            ],
        );
    }

    #[test]
    fn find_line_at_pos() {
        let test_data = TestData::new();
        let lines: Vec<_> = test_data
            .byte_positions()
            .iter()
            .map(|&pos| test_data.filemap.find_line_at_pos(pos))
            .collect();

        assert_eq!(
            lines,
            vec![
                Err(BytePosError::OutOfBounds {
                    given: BytePos(0),
                    span: test_data.filemap.span(),
                }),
                Ok(LineIndex(0)),
                Ok(LineIndex(0)),
                Ok(LineIndex(1)),
                Ok(LineIndex(1)),
                Ok(LineIndex(2)),
                Ok(LineIndex(2)),
                Ok(LineIndex(3)),
                Ok(LineIndex(3)),
                Ok(LineIndex(4)),
                Ok(LineIndex(4)),
                Ok(LineIndex(5)),
                Err(BytePosError::OutOfBounds {
                    given: BytePos(28),
                    span: test_data.filemap.span(),
                }),
            ],
        );
    }
}
<|endoftext|>


<commit_before>#[cfg(feature = "serde_macros")]
include!("botocore.rs");
#[cfg(not(feature = "serde_macros"))]
include!(concat!(env!("OUT_DIR"), "/botocore.rs"));
<commit_msg>Use correct file name when including botocore module source on nightly.<commit_after>#[cfg(feature = "serde_macros")]
include!("botocore.in.rs");
#[cfg(not(feature = "serde_macros"))]
include!(concat!(env!("OUT_DIR"), "/botocore.rs"));
<|endoftext|>


<commit_before>use common::memory::*;

use programs::common::*;

pub struct MemoryScheme;

impl SessionModule for MemoryScheme {
    fn scheme(&self) -> String {
        return "memory".to_string();
    }

    fn open(&mut self, url: &URL) -> Box<Resource>{
        let string = "Memory Used: ".to_string() + memory_used()/1024/1024 + " MB\n" + "Memory Free: " + memory_free()/1024/1024 + " MB";
        return box VecResource::new(ResourceType::File, string.to_utf8());
    }
}
<commit_msg>Show stack usage<commit_after>use common::memory::*;

use programs::common::*;

pub struct MemoryScheme;

fn stack_used() -> usize {
    unsafe{
        let stack: u32;
        asm!("" : "={esp}"(stack));
        return (0x200000 - stack) as usize;
    }
}

fn stack_free() -> usize {
    unsafe{
        let stack: u32;
        asm!("" : "={esp}"(stack));
        return (stack - 0x100000) as usize;
    }
}

impl SessionModule for MemoryScheme {
    fn scheme(&self) -> String {
        return "memory".to_string();
    }

    fn open(&mut self, url: &URL) -> Box<Resource>{
        let string = "Memory Used: ".to_string() + memory_used()/1024/1024 + " MB\n"
                   + "Memory Free: " + memory_free()/1024/1024 + " MB\n"
                   + "Stack Used: ".to_string() + stack_used()/1024 + " KB\n"
                   + "Stack Free: ".to_string() + stack_free()/1024 + " KB";
        return box VecResource::new(ResourceType::File, string.to_utf8());
    }
}
<|endoftext|>


<commit_before>use ast;
use ast::visit::{self, Visitor};
use class::*;
use ctxt::{Context, Fct, FctId, FctKind, FctSrc};
use error::msg::Msg;
use lexer::position::Position;
use mem;
use semck;
use ty::BuiltinType;

pub fn check<'ast>(ctxt: &mut Context<'ast>) {
    let mut clsck = ClsDefCheck {
        ctxt: ctxt,
        ast: ctxt.ast,
        cls_id: None,
    };

    clsck.check();
}

struct ClsDefCheck<'x, 'ast: 'x> {
    ctxt: &'x mut Context<'ast>,
    ast: &'ast ast::Ast,
    cls_id: Option<ClassId>,
}

impl<'x, 'ast> ClsDefCheck<'x, 'ast> {
    fn check(&mut self) {
        self.visit_ast(self.ast);
    }

    fn cls(&self) -> &Class<'ast> {
        self.ctxt.cls_by_id(self.cls_id.unwrap())
    }

    fn cls_mut(&mut self) -> &mut Class<'ast> {
        self.ctxt.cls_by_id_mut(self.cls_id.unwrap())
    }
}

impl<'x, 'ast> Visitor<'ast> for ClsDefCheck<'x, 'ast> {
    fn visit_class(&mut self, c: &'ast ast::Class) {
        self.cls_id = Some(*self.ctxt.cls_defs.get(&c.id).unwrap());

        visit::walk_class(self, c);

        self.cls_id = None;
    }

    fn visit_prop(&mut self, p: &'ast ast::Prop) {
        let ty = semck::read_type(self.ctxt, &p.data_type);

        for prop in &self.cls().props {
            if prop.name == p.name {
                let name = self.ctxt.interner.str(p.name).to_string();
                report(self.ctxt, p.pos, Msg::ShadowProp(name));
            }
        }

        let mut class = self.cls_mut();

        let offset = if ty.size() > 0 {
            mem::align_i32(class.size, ty.size())
        } else {
            class.size
        };

        let id = PropId(class.props.len());

        let prop = Prop {
            id: id,
            name: p.name,
            ty: ty,
            offset: offset,
        };

        class.size = offset + ty.size();
        class.props.push(prop);
    }

    fn visit_ctor(&mut self, f: &'ast ast::Function) {
        let clsid = self.cls_id.unwrap();

        let fct = Fct {
            id: FctId(0),
            name: f.name,
            params_types: Vec::new(),
            return_type: BuiltinType::Unit,
            owner_class: Some(clsid),
            ctor: true,
            initialized: false,
            kind: FctKind::Source(FctSrc::new(f)),
        };

        let fctid = self.ctxt.add_fct(fct);
        self.cls_mut().ctors.push(fctid);
    }

    fn visit_method(&mut self, f: &'ast ast::Function) {
        let fct = Fct {
            id: FctId(0),
            name: f.name,
            params_types: Vec::new(),
            return_type: BuiltinType::Unit,
            owner_class: Some(self.cls_id.unwrap()),
            ctor: false,
            initialized: false,
            kind: FctKind::Source(FctSrc::new(f)),
        };

        let fctid = self.ctxt.add_fct(fct);

        self.cls_mut().methods.push(fctid);
    }
}

fn report(ctxt: &Context, pos: Position, msg: Msg) {
    ctxt.diag.borrow_mut().report(pos, msg);
}

#[cfg(test)]
mod tests {
    use error::msg::Msg;
    use interner::Name;
    use mem;
    use semck::tests::*;

    fn class_size(code: &'static str) -> i32 {
        ok_with_test(code, |ctxt| {
            ctxt.classes[1].size
        })
    }

    #[test]
    fn test_class_size() {
        assert_eq!(mem::ptr_width(), class_size("class Foo"));
        assert_eq!(mem::ptr_width() + 4, class_size("class Foo(a: int)"));
        assert_eq!(2 * mem::ptr_width(), class_size("class Foo(a: Str)"));
    }

    #[test]
    fn test_multiple_definition() {
        err("class Foo class Foo", pos(1, 11), Msg::ShadowType("Foo".into()));
    }

    #[test]
    fn test_class_and_function() {
        err("fn Foo() {} class Foo", pos(1, 13), Msg::ShadowFunction("Foo".into()));
        err("class Foo fn Foo() {}", pos(1, 11), Msg::ShadowType("Foo".into()));
    }

    #[test]
    fn test_class_definition() {
        ok("class Foo");
        ok("class Foo()");
        ok("class Foo(a: int)");
        ok("class Foo(a: int, b:int)");
        ok("class Foo(a: Foo)");
        ok("class Foo(a: Bar) class Bar");
        err("class Foo(a: Unknown)", pos(1, 14), Msg::UnknownType(Name(2)));
        err("class Foo(a: int, a: int)", pos(1, 19), Msg::ShadowProp("a".to_string()));
    }
}
<commit_msg>fixed test for class size<commit_after>use ast;
use ast::visit::{self, Visitor};
use class::*;
use ctxt::{Context, Fct, FctId, FctKind, FctSrc};
use error::msg::Msg;
use lexer::position::Position;
use mem;
use semck;
use ty::BuiltinType;

pub fn check<'ast>(ctxt: &mut Context<'ast>) {
    let mut clsck = ClsDefCheck {
        ctxt: ctxt,
        ast: ctxt.ast,
        cls_id: None,
    };

    clsck.check();
}

struct ClsDefCheck<'x, 'ast: 'x> {
    ctxt: &'x mut Context<'ast>,
    ast: &'ast ast::Ast,
    cls_id: Option<ClassId>,
}

impl<'x, 'ast> ClsDefCheck<'x, 'ast> {
    fn check(&mut self) {
        self.visit_ast(self.ast);
    }

    fn cls(&self) -> &Class<'ast> {
        self.ctxt.cls_by_id(self.cls_id.unwrap())
    }

    fn cls_mut(&mut self) -> &mut Class<'ast> {
        self.ctxt.cls_by_id_mut(self.cls_id.unwrap())
    }
}

impl<'x, 'ast> Visitor<'ast> for ClsDefCheck<'x, 'ast> {
    fn visit_class(&mut self, c: &'ast ast::Class) {
        self.cls_id = Some(*self.ctxt.cls_defs.get(&c.id).unwrap());

        visit::walk_class(self, c);

        self.cls_id = None;
    }

    fn visit_prop(&mut self, p: &'ast ast::Prop) {
        let ty = semck::read_type(self.ctxt, &p.data_type);

        for prop in &self.cls().props {
            if prop.name == p.name {
                let name = self.ctxt.interner.str(p.name).to_string();
                report(self.ctxt, p.pos, Msg::ShadowProp(name));
            }
        }

        let mut class = self.cls_mut();

        let offset = if ty.size() > 0 {
            mem::align_i32(class.size, ty.size())
        } else {
            class.size
        };

        let id = PropId(class.props.len());

        let prop = Prop {
            id: id,
            name: p.name,
            ty: ty,
            offset: offset,
        };

        class.size = offset + ty.size();
        class.props.push(prop);
    }

    fn visit_ctor(&mut self, f: &'ast ast::Function) {
        let clsid = self.cls_id.unwrap();

        let fct = Fct {
            id: FctId(0),
            name: f.name,
            params_types: Vec::new(),
            return_type: BuiltinType::Unit,
            owner_class: Some(clsid),
            ctor: true,
            initialized: false,
            kind: FctKind::Source(FctSrc::new(f)),
        };

        let fctid = self.ctxt.add_fct(fct);
        self.cls_mut().ctors.push(fctid);
    }

    fn visit_method(&mut self, f: &'ast ast::Function) {
        let fct = Fct {
            id: FctId(0),
            name: f.name,
            params_types: Vec::new(),
            return_type: BuiltinType::Unit,
            owner_class: Some(self.cls_id.unwrap()),
            ctor: false,
            initialized: false,
            kind: FctKind::Source(FctSrc::new(f)),
        };

        let fctid = self.ctxt.add_fct(fct);

        self.cls_mut().methods.push(fctid);
    }
}

fn report(ctxt: &Context, pos: Position, msg: Msg) {
    ctxt.diag.borrow_mut().report(pos, msg);
}

#[cfg(test)]
mod tests {
    use error::msg::Msg;
    use interner::Name;
    use mem;
    use semck::tests::*;

    fn class_size(code: &'static str) -> i32 {
        ok_with_test(code, |ctxt| {
            ctxt.classes[4].size
        })
    }

    #[test]
    fn test_class_size() {
        assert_eq!(mem::ptr_width(), class_size("class Foo"));
        assert_eq!(mem::ptr_width() + 4, class_size("class Foo(a: int)"));
        assert_eq!(2 * mem::ptr_width(), class_size("class Foo(a: Str)"));
    }

    #[test]
    fn test_multiple_definition() {
        err("class Foo class Foo", pos(1, 11), Msg::ShadowType("Foo".into()));
    }

    #[test]
    fn test_class_and_function() {
        err("fn Foo() {} class Foo", pos(1, 13), Msg::ShadowFunction("Foo".into()));
        err("class Foo fn Foo() {}", pos(1, 11), Msg::ShadowType("Foo".into()));
    }

    #[test]
    fn test_class_definition() {
        ok("class Foo");
        ok("class Foo()");
        ok("class Foo(a: int)");
        ok("class Foo(a: int, b:int)");
        ok("class Foo(a: Foo)");
        ok("class Foo(a: Bar) class Bar");
        err("class Foo(a: Unknown)", pos(1, 14), Msg::UnknownType(Name(2)));
        err("class Foo(a: int, a: int)", pos(1, 19), Msg::ShadowProp("a".to_string()));
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeSet;

use edgeql_parser::tokenizer::{TokenStream, Kind};
use edgeql_parser::position::Pos;
use num_bigint::{BigInt, ToBigInt};
use bigdecimal::BigDecimal;
use crate::tokenizer::{CowToken, decode_string};

#[derive(Debug, PartialEq)]
pub enum Value {
    Str(String),
    Int(i64),
    Float(f64),
    BigInt(BigInt),
    Decimal(BigDecimal),
}

#[derive(Debug, PartialEq)]
pub struct Variable {
    pub value: Value,
}

#[derive(Debug)]
pub struct Entry<'a> {
    pub key: String,
    pub tokens: Vec<CowToken<'a>>,
    pub variables: Vec<Variable>,
    pub end_pos: Pos,
    pub named_args: bool,
    pub first_arg: Option<usize>,
}

#[derive(Debug)]
pub enum Error {
    Tokenizer(String, Pos),
    Assertion(String, Pos),
}

fn push_var<'x>(res: &mut Vec<CowToken<'x>>, typ: &'x str, var: String,
    start: Pos, end: Pos)
{
    res.push(CowToken {kind: Kind::OpenParen, value: "(".into(), start, end});
    res.push(CowToken {kind: Kind::Less, value: "<".into(), start, end});
    res.push(CowToken {kind: Kind::Ident, value: typ.into(), start, end});
    res.push(CowToken {kind: Kind::Greater, value: ">".into(), start, end});
    res.push(CowToken {kind: Kind::Argument, value: var.into(), start, end});
    res.push(CowToken {kind: Kind::CloseParen, value: ")".into(), start, end});
}

fn scan_vars<'x, 'y: 'x, I>(tokens: I) -> Option<(bool, usize)>
    where I: IntoIterator<Item=&'x CowToken<'y>>,
{
    let mut max_visited = None::<usize>;
    let mut names = BTreeSet::new();
    for t in tokens {
        if t.kind == Kind::Argument {
            if let Ok(v) = t.value[1..].parse() {
                if max_visited.map(|old| v > old).unwrap_or(true) {
                    max_visited = Some(v);
                }
            } else {
                names.insert(&t.value[..]);
            }
        }
    }
    if names.is_empty() {
        let next = max_visited.map(|x| x.checked_add(1)).unwrap_or(Some(0))?;
        Some((false, next))
    } else if max_visited.is_some() {
        return None  // mixed arguments
    } else {
        Some((true, names.len()))
    }
}

pub fn normalize<'x>(text: &'x str)
    -> Result<Entry<'x>, Error>
{
    use combine::easy::Error::*;
    let mut token_stream = TokenStream::new(&text);
    let mut tokens = Vec::new();
    for res in &mut token_stream {
        match res {
            Ok(t) => tokens.push(CowToken::from(t)),
            Err(Unexpected(s)) => {
                return Err(Error::Tokenizer(
                    s.to_string(), token_stream.current_pos()));
            }
            Err(e) => {
                return Err(Error::Tokenizer(
                    e.to_string(), token_stream.current_pos()));
            }
        }
    }
    let end_pos = token_stream.current_pos();
    let (named_args, var_idx) = match scan_vars(&tokens) {
        Some(pair) => pair,
        None => {
            // don't extract from invalid query, let python code do its work
            return Ok(Entry {
                key: serialize_tokens(&tokens),
                tokens,
                variables: Vec::new(),
                end_pos,
                named_args: false,
                first_arg: None,
            });
        }
    };
    let mut rewritten_tokens = Vec::with_capacity(tokens.len());
    let mut variables = Vec::new();
    let next_var = |num: usize| {
        if named_args {
            format!("$__edb_arg_{}", var_idx + num)
        } else {
            format!("${}", var_idx + num)
        }
    };
    for tok in &tokens {
        match tok.kind {
            Kind::IntConst
            // Don't replace `.12` because this is a tuple access
            if !matches!(rewritten_tokens.last(),
                Some(CowToken { kind: Kind::Dot, .. }))
            // Don't replace 'LIMIT 1' as a special case
            && (tok.value != "1"
                || !matches!(rewritten_tokens.last(),
                    Some(CowToken { kind: Kind::Keyword, ref value, .. })
                    if value.eq_ignore_ascii_case("LIMIT")))
            && tok.value != "9223372036854775808"
            => {
                push_var(&mut rewritten_tokens, "__std__::int64",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Int(tok.value.replace("_", "").parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse integer: {}", e),
                            tok.start))?),
                });
                continue;
            }
            Kind::FloatConst => {
                push_var(&mut rewritten_tokens, "__std__::float64",
                    next_var(variables.len()),
                    tok.start, tok.end);
                let value = tok.value.replace("_", "").parse()
                    .map_err(|e| Error::Tokenizer(
                        format!("can't parse std::float64: {}", e),
                        tok.start))?;
                if value == f64::INFINITY || value == -f64::INFINITY {
                    return Err(Error::Tokenizer(
                        format!("number is out of range for std::float64"),
                        tok.start));
                }
                variables.push(Variable {
                    value: Value::Float(value),
                });
                continue;
            }
            Kind::BigIntConst => {
                push_var(&mut rewritten_tokens, "__std__::bigint",
                    next_var(variables.len()),
                    tok.start, tok.end);
                let dec: BigDecimal = tok.value[..tok.value.len()-1]
                        .replace("_", "").parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse bigint: {}", e),
                            tok.start))?;
                variables.push(Variable {
                    value: Value::BigInt(dec.to_bigint()
                        .ok_or_else(|| Error::Assertion(
                            format!("number is not integer"),
                            tok.start))?),
                });
                continue;
            }
            Kind::DecimalConst => {
                push_var(&mut rewritten_tokens, "__std__::decimal",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Decimal(
                        tok.value[..tok.value.len()-1]
                        .replace("_", "")
                        .parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse decimal: {}", e),
                            tok.start))?),
                });
                continue;
            }
            Kind::Str => {
                push_var(&mut rewritten_tokens, "__std__::str",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Str(decode_string(&tok.value)
                        .map_err(|e| Error::Tokenizer(
                            format!("can't unquote string: {}", e),
                            tok.start))?.into()),
                });
                continue;
            }
            Kind::Keyword
            if (matches!(&(&tok.value[..].to_uppercase())[..],
                "CONFIGURE"|"CREATE"|"ALTER"|"DROP"))
            => {
                return Ok(Entry {
                    key: serialize_tokens(&tokens),
                    tokens,
                    variables: Vec::new(),
                    end_pos,
                    named_args: false,
                    first_arg: None,
                });
            }
            _ => rewritten_tokens.push(tok.clone()),
        }
    }
    return Ok(Entry {
        named_args,
        first_arg: if variables.is_empty() { None } else { Some(var_idx) },
        key: serialize_tokens(&rewritten_tokens[..]),
        tokens: rewritten_tokens,
        variables,
        end_pos,
    });
}

fn is_operator(token: &CowToken) -> bool {
    use edgeql_parser::tokenizer::Kind::*;
    match token.kind {
        | Assign
        | SubAssign
        | AddAssign
        | Arrow
        | Coalesce
        | Namespace
        | ForwardLink
        | BackwardLink
        | FloorDiv
        | Concat
        | GreaterEq
        | LessEq
        | NotEq
        | NotDistinctFrom
        | DistinctFrom
        | Comma
        | OpenParen
        | CloseParen
        | OpenBracket
        | CloseBracket
        | OpenBrace
        | CloseBrace
        | Dot
        | Semicolon
        | Colon
        | Add
        | Sub
        | Mul
        | Div
        | Modulo
        | Pow
        | Less
        | Greater
        | Eq
        | Ampersand
        | Pipe
        | At
        => true,
        | DecimalConst
        | FloatConst
        | IntConst
        | BigIntConst
        | BinStr
        | Argument
        | Str
        | BacktickName
        | Keyword
        | Ident
        => false,
    }
}

fn serialize_tokens(tokens: &[CowToken<'_>]) -> String {
    use edgeql_parser::tokenizer::Kind::Argument;

    let mut buf = String::new();
    let mut needs_space = false;
    for token in tokens {
        if needs_space && !is_operator(token) && token.kind != Argument {
            buf.push(' ');
        }
        buf.push_str(&token.value);
        needs_space = !is_operator(token);
    }
    return buf;
}

#[cfg(test)]
mod test {
    use super::scan_vars;
    use combine::{StreamOnce, Positioned, easy::Error};
    use edgeql_parser::tokenizer::{TokenStream};
    use edgeql_parser::position::Pos;
    use crate::tokenizer::{CowToken};

    fn tokenize<'x>(s: &'x str) -> Vec<CowToken<'x>> {
        let mut r = Vec::new();
        let mut s = TokenStream::new(s);
        loop {
            match s.uncons() {
                Ok(x) => r.push(CowToken {
                    kind: x.kind,
                    value: x.value.into(),
                    start: Pos { line: 0, column: 0, offset: 0 },
                    end: Pos { line: 0, column: 0, offset: 0 },
                }),
                Err(ref e) if e == &Error::end_of_input() => break,
                Err(e) => panic!("Parse error at {}: {}", s.position(), e),
            }
        }
        return r;
    }

    #[test]
    fn none() {
        assert_eq!(scan_vars(&tokenize("SELECT 1+1")).unwrap(), (false, 0));
    }

    #[test]
    fn numeric() {
        assert_eq!(scan_vars(&tokenize("$0 $1 $2")).unwrap(), (false, 3));
        assert_eq!(scan_vars(&tokenize("$2 $3 $2")).unwrap(), (false, 4));
        assert_eq!(scan_vars(&tokenize("$0 $0 $0")).unwrap(), (false, 1));
        assert_eq!(scan_vars(&tokenize("$10 $100")).unwrap(), (false, 101));
    }

    #[test]
    fn named() {
        assert_eq!(scan_vars(&tokenize("$a")).unwrap(), (true, 1));
        assert_eq!(scan_vars(&tokenize("$b $c $d")).unwrap(), (true, 3));
        assert_eq!(scan_vars(&tokenize("$b $c $b")).unwrap(), (true, 2));
        assert_eq!(scan_vars(&tokenize("$a $b $b $a $c $xx")).unwrap(),
            (true, 4));
    }

    #[test]
    fn mixed() {
        assert_eq!(scan_vars(&tokenize("$a $0")), None);
        assert_eq!(scan_vars(&tokenize("$0 $a")), None);
        assert_eq!(scan_vars(&tokenize("$b $c $100")), None);
        assert_eq!(scan_vars(&tokenize("$10 $xx $yy")), None);
    }

}
<commit_msg>Exempt `START MIGRATION` from constraint extraction normalization (#1547)<commit_after>use std::collections::BTreeSet;

use edgeql_parser::tokenizer::{TokenStream, Kind};
use edgeql_parser::position::Pos;
use num_bigint::{BigInt, ToBigInt};
use bigdecimal::BigDecimal;
use crate::tokenizer::{CowToken, decode_string};

#[derive(Debug, PartialEq)]
pub enum Value {
    Str(String),
    Int(i64),
    Float(f64),
    BigInt(BigInt),
    Decimal(BigDecimal),
}

#[derive(Debug, PartialEq)]
pub struct Variable {
    pub value: Value,
}

#[derive(Debug)]
pub struct Entry<'a> {
    pub key: String,
    pub tokens: Vec<CowToken<'a>>,
    pub variables: Vec<Variable>,
    pub end_pos: Pos,
    pub named_args: bool,
    pub first_arg: Option<usize>,
}

#[derive(Debug)]
pub enum Error {
    Tokenizer(String, Pos),
    Assertion(String, Pos),
}

fn push_var<'x>(res: &mut Vec<CowToken<'x>>, typ: &'x str, var: String,
    start: Pos, end: Pos)
{
    res.push(CowToken {kind: Kind::OpenParen, value: "(".into(), start, end});
    res.push(CowToken {kind: Kind::Less, value: "<".into(), start, end});
    res.push(CowToken {kind: Kind::Ident, value: typ.into(), start, end});
    res.push(CowToken {kind: Kind::Greater, value: ">".into(), start, end});
    res.push(CowToken {kind: Kind::Argument, value: var.into(), start, end});
    res.push(CowToken {kind: Kind::CloseParen, value: ")".into(), start, end});
}

fn scan_vars<'x, 'y: 'x, I>(tokens: I) -> Option<(bool, usize)>
    where I: IntoIterator<Item=&'x CowToken<'y>>,
{
    let mut max_visited = None::<usize>;
    let mut names = BTreeSet::new();
    for t in tokens {
        if t.kind == Kind::Argument {
            if let Ok(v) = t.value[1..].parse() {
                if max_visited.map(|old| v > old).unwrap_or(true) {
                    max_visited = Some(v);
                }
            } else {
                names.insert(&t.value[..]);
            }
        }
    }
    if names.is_empty() {
        let next = max_visited.map(|x| x.checked_add(1)).unwrap_or(Some(0))?;
        Some((false, next))
    } else if max_visited.is_some() {
        return None  // mixed arguments
    } else {
        Some((true, names.len()))
    }
}

pub fn normalize<'x>(text: &'x str)
    -> Result<Entry<'x>, Error>
{
    use combine::easy::Error::*;
    let mut token_stream = TokenStream::new(&text);
    let mut tokens = Vec::new();
    for res in &mut token_stream {
        match res {
            Ok(t) => tokens.push(CowToken::from(t)),
            Err(Unexpected(s)) => {
                return Err(Error::Tokenizer(
                    s.to_string(), token_stream.current_pos()));
            }
            Err(e) => {
                return Err(Error::Tokenizer(
                    e.to_string(), token_stream.current_pos()));
            }
        }
    }
    let end_pos = token_stream.current_pos();
    let (named_args, var_idx) = match scan_vars(&tokens) {
        Some(pair) => pair,
        None => {
            // don't extract from invalid query, let python code do its work
            return Ok(Entry {
                key: serialize_tokens(&tokens),
                tokens,
                variables: Vec::new(),
                end_pos,
                named_args: false,
                first_arg: None,
            });
        }
    };
    let mut rewritten_tokens = Vec::with_capacity(tokens.len());
    let mut variables = Vec::new();
    let next_var = |num: usize| {
        if named_args {
            format!("$__edb_arg_{}", var_idx + num)
        } else {
            format!("${}", var_idx + num)
        }
    };
    for tok in &tokens {
        match tok.kind {
            Kind::IntConst
            // Don't replace `.12` because this is a tuple access
            if !matches!(rewritten_tokens.last(),
                Some(CowToken { kind: Kind::Dot, .. }))
            // Don't replace 'LIMIT 1' as a special case
            && (tok.value != "1"
                || !matches!(rewritten_tokens.last(),
                    Some(CowToken { kind: Kind::Keyword, ref value, .. })
                    if value.eq_ignore_ascii_case("LIMIT")))
            && tok.value != "9223372036854775808"
            => {
                push_var(&mut rewritten_tokens, "__std__::int64",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Int(tok.value.replace("_", "").parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse integer: {}", e),
                            tok.start))?),
                });
                continue;
            }
            Kind::FloatConst => {
                push_var(&mut rewritten_tokens, "__std__::float64",
                    next_var(variables.len()),
                    tok.start, tok.end);
                let value = tok.value.replace("_", "").parse()
                    .map_err(|e| Error::Tokenizer(
                        format!("can't parse std::float64: {}", e),
                        tok.start))?;
                if value == f64::INFINITY || value == -f64::INFINITY {
                    return Err(Error::Tokenizer(
                        format!("number is out of range for std::float64"),
                        tok.start));
                }
                variables.push(Variable {
                    value: Value::Float(value),
                });
                continue;
            }
            Kind::BigIntConst => {
                push_var(&mut rewritten_tokens, "__std__::bigint",
                    next_var(variables.len()),
                    tok.start, tok.end);
                let dec: BigDecimal = tok.value[..tok.value.len()-1]
                        .replace("_", "").parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse bigint: {}", e),
                            tok.start))?;
                variables.push(Variable {
                    value: Value::BigInt(dec.to_bigint()
                        .ok_or_else(|| Error::Assertion(
                            format!("number is not integer"),
                            tok.start))?),
                });
                continue;
            }
            Kind::DecimalConst => {
                push_var(&mut rewritten_tokens, "__std__::decimal",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Decimal(
                        tok.value[..tok.value.len()-1]
                        .replace("_", "")
                        .parse()
                        .map_err(|e| Error::Tokenizer(
                            format!("can't parse decimal: {}", e),
                            tok.start))?),
                });
                continue;
            }
            Kind::Str => {
                push_var(&mut rewritten_tokens, "__std__::str",
                    next_var(variables.len()),
                    tok.start, tok.end);
                variables.push(Variable {
                    value: Value::Str(decode_string(&tok.value)
                        .map_err(|e| Error::Tokenizer(
                            format!("can't unquote string: {}", e),
                            tok.start))?.into()),
                });
                continue;
            }
            Kind::Keyword
            if (matches!(&(&tok.value[..].to_uppercase())[..],
                "CONFIGURE"|"CREATE"|"ALTER"|"DROP"|"START"))
            => {
                return Ok(Entry {
                    key: serialize_tokens(&tokens),
                    tokens,
                    variables: Vec::new(),
                    end_pos,
                    named_args: false,
                    first_arg: None,
                });
            }
            _ => rewritten_tokens.push(tok.clone()),
        }
    }
    return Ok(Entry {
        named_args,
        first_arg: if variables.is_empty() { None } else { Some(var_idx) },
        key: serialize_tokens(&rewritten_tokens[..]),
        tokens: rewritten_tokens,
        variables,
        end_pos,
    });
}

fn is_operator(token: &CowToken) -> bool {
    use edgeql_parser::tokenizer::Kind::*;
    match token.kind {
        | Assign
        | SubAssign
        | AddAssign
        | Arrow
        | Coalesce
        | Namespace
        | ForwardLink
        | BackwardLink
        | FloorDiv
        | Concat
        | GreaterEq
        | LessEq
        | NotEq
        | NotDistinctFrom
        | DistinctFrom
        | Comma
        | OpenParen
        | CloseParen
        | OpenBracket
        | CloseBracket
        | OpenBrace
        | CloseBrace
        | Dot
        | Semicolon
        | Colon
        | Add
        | Sub
        | Mul
        | Div
        | Modulo
        | Pow
        | Less
        | Greater
        | Eq
        | Ampersand
        | Pipe
        | At
        => true,
        | DecimalConst
        | FloatConst
        | IntConst
        | BigIntConst
        | BinStr
        | Argument
        | Str
        | BacktickName
        | Keyword
        | Ident
        => false,
    }
}

fn serialize_tokens(tokens: &[CowToken<'_>]) -> String {
    use edgeql_parser::tokenizer::Kind::Argument;

    let mut buf = String::new();
    let mut needs_space = false;
    for token in tokens {
        if needs_space && !is_operator(token) && token.kind != Argument {
            buf.push(' ');
        }
        buf.push_str(&token.value);
        needs_space = !is_operator(token);
    }
    return buf;
}

#[cfg(test)]
mod test {
    use super::scan_vars;
    use combine::{StreamOnce, Positioned, easy::Error};
    use edgeql_parser::tokenizer::{TokenStream};
    use edgeql_parser::position::Pos;
    use crate::tokenizer::{CowToken};

    fn tokenize<'x>(s: &'x str) -> Vec<CowToken<'x>> {
        let mut r = Vec::new();
        let mut s = TokenStream::new(s);
        loop {
            match s.uncons() {
                Ok(x) => r.push(CowToken {
                    kind: x.kind,
                    value: x.value.into(),
                    start: Pos { line: 0, column: 0, offset: 0 },
                    end: Pos { line: 0, column: 0, offset: 0 },
                }),
                Err(ref e) if e == &Error::end_of_input() => break,
                Err(e) => panic!("Parse error at {}: {}", s.position(), e),
            }
        }
        return r;
    }

    #[test]
    fn none() {
        assert_eq!(scan_vars(&tokenize("SELECT 1+1")).unwrap(), (false, 0));
    }

    #[test]
    fn numeric() {
        assert_eq!(scan_vars(&tokenize("$0 $1 $2")).unwrap(), (false, 3));
        assert_eq!(scan_vars(&tokenize("$2 $3 $2")).unwrap(), (false, 4));
        assert_eq!(scan_vars(&tokenize("$0 $0 $0")).unwrap(), (false, 1));
        assert_eq!(scan_vars(&tokenize("$10 $100")).unwrap(), (false, 101));
    }

    #[test]
    fn named() {
        assert_eq!(scan_vars(&tokenize("$a")).unwrap(), (true, 1));
        assert_eq!(scan_vars(&tokenize("$b $c $d")).unwrap(), (true, 3));
        assert_eq!(scan_vars(&tokenize("$b $c $b")).unwrap(), (true, 2));
        assert_eq!(scan_vars(&tokenize("$a $b $b $a $c $xx")).unwrap(),
            (true, 4));
    }

    #[test]
    fn mixed() {
        assert_eq!(scan_vars(&tokenize("$a $0")), None);
        assert_eq!(scan_vars(&tokenize("$0 $a")), None);
        assert_eq!(scan_vars(&tokenize("$b $c $100")), None);
        assert_eq!(scan_vars(&tokenize("$10 $xx $yy")), None);
    }

}
<|endoftext|>


<commit_before>use std::io::{BufReader, BufRead};
use std::fs::File;
use rand::Rng;

const K: usize = 10;

pub fn reservoir_sample<R: Rng>(f: File, mut rng: R) -> String {
    let reader = BufReader::new(f);

    let mut reservoir: [String;K] = Default::default();

    for (i, line) in reader.lines().enumerate() {
        let l = line.unwrap();

        if i < K {
            reservoir[i] = l;
        } else {
            let j = rng.gen_range(0, i);
            if j < K {
                reservoir[j] = l;
            }
        }
    }

    let n = rng.gen_range(0, K);

    reservoir[n].clone()
}
<commit_msg>Change reservoir size to 5<commit_after>use std::io::{BufReader, BufRead};
use std::fs::File;
use rand::Rng;

const K: usize = 5;

pub fn reservoir_sample<R: Rng>(f: File, mut rng: R) -> String {
    let reader = BufReader::new(f);

    let mut reservoir: [String;K] = Default::default();

    for (i, line) in reader.lines().enumerate() {
        let l = line.unwrap();

        if i < K {
            reservoir[i] = l;
        } else {
            let j = rng.gen_range(0, i);
            if j < K {
                reservoir[j] = l;
            }
        }
    }

    let n = rng.gen_range(0, K);

    reservoir[n].clone()
}
<|endoftext|>