<commit_before>#![feature(collections)]
#![feature(convert)]

use std::borrow::Cow;
use std::borrow::Cow::{Borrowed};

pub fn compile(sass: &str, style: &str) -> Result<String, &'static str> {
    let mut st = SassTokenizer::new(&sass);

    // while let Some(foo) = st.next() {
    //     println!("{:?}", foo);
    // }

    match style {
        "nested"     => Ok(nested_output(&mut st)),
        "compressed" => Ok(compressed_output(&mut st)),
        "expanded"   => Ok(expanded_output(&mut st)),
    //     "compact"    => Ok(parsed.compact(&sp)),
        _            => Err("Unknown style:. Please specify one of nested, compressed, expanded, or compact."),
    }
}

pub fn nested_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{} {{", name),
            Event::Property(name, value) => format!("\n  {}: {};", name, value),
            Event::End(_) => format!(" }}\n"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

pub fn compressed_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{}{{", name),
            Event::Property(name, value) => format!("{}:{}", name, value),
            Event::End(_) => format!("}}"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

pub fn expanded_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{} {{", name),
            Event::Property(name, value) => format!("\n  {}: {};", name, value),
            Event::End(_) => format!("\n}}"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

#[derive(PartialEq, Debug)]
pub enum State {
    StartRule,
    InSelectors,
    InProperties,
}

#[derive(Debug)]
pub enum Rule {
    SassRule,
}

#[derive(Debug)]
pub enum Event<'a> {
    Start(Rule),
    End(Rule),
    Selector(Cow<'a, str>),
    Property(Cow<'a, str>, Cow<'a, str>),
}

#[derive(Debug)]
pub struct SassTokenizer<'a> {
    sass: &'a str,
    offset: usize,
    stack: Vec<Rule>,
    state: State,
}

impl<'a> SassTokenizer<'a> {
    pub fn new(sass: &'a str) -> SassTokenizer<'a> {
        SassTokenizer {
            sass: &sass,
            offset: 0,
            stack: Vec::new(),
            state: State::StartRule,
        }
    }

    pub fn start_rule(&mut self) -> Option<Event<'a>> {
        self.state = State::InSelectors;
        self.stack.push(Rule::SassRule);

        Some(Event::Start(Rule::SassRule))
    }

    fn end(&mut self) -> Event<'a> {
        let rule = self.stack.pop().unwrap();
        self.state = State::StartRule;
        Event::End(rule)
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let i = self.offset;
        self.offset += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
    }

    pub fn next_property(&mut self) -> Event<'a> {
        self.skip_leading_whitespace();

        let bytes = self.sass.as_bytes();
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = bytes[i];
        if c == b'}' {
            self.offset += 1;
            return self.end()
        }

        while i < limit {
            match bytes[i..limit].iter().position(|&c| c == b':') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Event::Property(
                    Borrowed(&self.sass[name_beginning..name_end]),
                    Borrowed(&self.sass[value_beginning..value_end])
                )

            }
        }
        self.offset = self.sass.len();
        Event::Property(Borrowed(""), Borrowed(""))
    }

    pub fn next_selector(&mut self) -> Event<'a> {
        let bytes = self.sass.as_bytes();
        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match bytes[i..limit].iter().position(|&c| c == b',' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = bytes[i];

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        self.state = State::InProperties;
                    }
                    self.offset = i + 1;
                    return Event::Selector(Borrowed(&self.sass[beginning..end]));
                }
            }

            self.offset = i;
            if i > beginning {
                return Event::Selector(Borrowed(&self.sass[beginning..i]))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Event::Selector(Borrowed(&self.sass[beginning..i]))
        } else {
            self.end()
        }
    }
}

impl<'a> Iterator for SassTokenizer<'a> {
    type Item = Event<'a>;

    fn next(&mut self) -> Option<Event<'a>> {
        if self.offset < self.sass.len() {
            match self.state {
                State::StartRule => {
                    let ret = self.start_rule();
                    if ret.is_some() {
                        return ret
                    }
                },
                State::InSelectors => return Some(self.next_selector()),
                State::InProperties => return Some(self.next_property()),
            }
        }
        None
    }
}

pub fn char_at(s: &str, byte: usize) -> char {
    s[byte..].chars().next().unwrap()
}

pub fn is_ascii_whitespace(c: u8) -> bool {
    c == b'\n' || c == b'\r' || is_ascii_whitespace_no_nl(c)
}

pub fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
pub fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}
<commit_msg>And now I'm back to where I was a few days ago!!!!<commit_after>#![feature(collections)]
#![feature(convert)]

use std::borrow::Cow;
use std::borrow::Cow::{Borrowed};

pub fn compile(sass: &str, style: &str) -> Result<String, &'static str> {
    let mut st = SassTokenizer::new(&sass);

    // while let Some(foo) = st.next() {
    //     println!("{:?}", foo);
    // }

    match style {
        "nested"     => Ok(nested_output(&mut st)),
        "compressed" => Ok(compressed_output(&mut st)),
        "expanded"   => Ok(expanded_output(&mut st)),
        "compact"    => Ok(compact_output(&mut st)),
        _            => Err("Unknown style:. Please specify one of nested, compressed, expanded, or compact."),
    }
}

pub fn nested_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{} {{", name),
            Event::Property(name, value) => format!("\n  {}: {};", name, value),
            Event::End(_) => format!(" }}\n"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

pub fn compressed_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{}{{", name),
            Event::Property(name, value) => format!("{}:{}", name, value),
            Event::End(_) => format!("}}"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

pub fn expanded_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{} {{", name),
            Event::Property(name, value) => format!("\n  {}: {};", name, value),
            Event::End(_) => format!("\n}}"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

pub fn compact_output(tokenizer: &mut SassTokenizer) -> String {
    let mut output =  String::from_str("");
    while let Some(token) = tokenizer.next() {
        let print_token = match token {
            Event::Start(_) => continue,
            Event::Selector(name) => format!("{} {{", name),
            Event::Property(name, value) => format!(" {}: {};", name, value),
            Event::End(_) => format!(" }}"),
        };
        output.push_str(print_token.as_str());
    }
    output
}

#[derive(PartialEq, Debug)]
pub enum State {
    StartRule,
    InSelectors,
    InProperties,
}

#[derive(Debug)]
pub enum Rule {
    SassRule,
}

#[derive(Debug)]
pub enum Event<'a> {
    Start(Rule),
    End(Rule),
    Selector(Cow<'a, str>),
    Property(Cow<'a, str>, Cow<'a, str>),
}

#[derive(Debug)]
pub struct SassTokenizer<'a> {
    sass: &'a str,
    offset: usize,
    stack: Vec<Rule>,
    state: State,
}

impl<'a> SassTokenizer<'a> {
    pub fn new(sass: &'a str) -> SassTokenizer<'a> {
        SassTokenizer {
            sass: &sass,
            offset: 0,
            stack: Vec::new(),
            state: State::StartRule,
        }
    }

    pub fn start_rule(&mut self) -> Option<Event<'a>> {
        self.state = State::InSelectors;
        self.stack.push(Rule::SassRule);

        Some(Event::Start(Rule::SassRule))
    }

    fn end(&mut self) -> Event<'a> {
        let rule = self.stack.pop().unwrap();
        self.state = State::StartRule;
        Event::End(rule)
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let i = self.offset;
        self.offset += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
    }

    pub fn next_property(&mut self) -> Event<'a> {
        self.skip_leading_whitespace();

        let bytes = self.sass.as_bytes();
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = bytes[i];
        if c == b'}' {
            self.offset += 1;
            return self.end()
        }

        while i < limit {
            match bytes[i..limit].iter().position(|&c| c == b':') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Event::Property(
                    Borrowed(&self.sass[name_beginning..name_end]),
                    Borrowed(&self.sass[value_beginning..value_end])
                )

            }
        }
        self.offset = self.sass.len();
        Event::Property(Borrowed(""), Borrowed(""))
    }

    pub fn next_selector(&mut self) -> Event<'a> {
        let bytes = self.sass.as_bytes();
        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match bytes[i..limit].iter().position(|&c| c == b',' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = bytes[i];

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        self.state = State::InProperties;
                    }
                    self.offset = i + 1;
                    return Event::Selector(Borrowed(&self.sass[beginning..end]));
                }
            }

            self.offset = i;
            if i > beginning {
                return Event::Selector(Borrowed(&self.sass[beginning..i]))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Event::Selector(Borrowed(&self.sass[beginning..i]))
        } else {
            self.end()
        }
    }
}

impl<'a> Iterator for SassTokenizer<'a> {
    type Item = Event<'a>;

    fn next(&mut self) -> Option<Event<'a>> {
        if self.offset < self.sass.len() {
            match self.state {
                State::StartRule => {
                    let ret = self.start_rule();
                    if ret.is_some() {
                        return ret
                    }
                },
                State::InSelectors => return Some(self.next_selector()),
                State::InProperties => return Some(self.next_property()),
            }
        }
        None
    }
}

pub fn char_at(s: &str, byte: usize) -> char {
    s[byte..].chars().next().unwrap()
}

pub fn is_ascii_whitespace(c: u8) -> bool {
    c == b'\n' || c == b'\r' || is_ascii_whitespace_no_nl(c)
}

pub fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
pub fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}
<|endoftext|>


<commit_before>//! The Public Suffix List Client Library
//!
//! This library allows you to easily and accurately validate
//! or break down any given domain name.
//!
//! ## Examples
//!
//! ```
//! extern crate publicsuffix;
//!
//! use publicsuffix::List;
//!
//! fn main() {
//!     let list = List::fetch().unwrap();
//!
//!     let domain = list.parse_domain("www.example.com").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("example.com"));
//!
//!     let domain = list.parse_domain("www.食狮.中国").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("食狮.中国"));
//!
//!     let domain = list.parse_domain("www.xn--85x722f.xn--55qx5d.cn").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("xn--85x722f.xn--55qx5d.cn"));
//!
//!     let domain = list.parse_domain("a.b.domain.biz").unwrap();
//!     let suffix = domain.suffix().unwrap();
//!     assert_eq!(suffix, &String::from("biz"));
//!
//!     let domain = list.parse_domain("a.b.example.uk.com").unwrap();
//!     let suffix = domain.suffix().unwrap();
//!     assert_eq!(suffix, &String::from("uk.com"));
//! }

#![recursion_limit = "1024"]

#[macro_use]
extern crate error_chain;
extern crate reqwest;
extern crate regex;
extern crate idna;

pub mod errors;

#[cfg(test)]
mod tests;

use std::io::Read;
use std::fs::File;
use std::path::Path;
use std::collections::HashMap;

use errors::*;
use regex::Regex;
use reqwest::IntoUrl;
use idna::{domain_to_unicode, domain_to_ascii};

#[derive(Debug)]
struct Suffix {
    rule: String,
    typ: Type,
}

/// Stores the public suffix list
///
/// You can use the methods, `fetch`, `from_url` or `from_path` to build the list.
/// If you are using this in a long running server it's recommended you use either
/// `fetch` or `from_url` to download updates at least once a week.
#[derive(Debug)]
pub struct List {
    rules: HashMap<String, Vec<Suffix>>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
enum Type {
    Icann,
    Private,
}

/// Holds information about a particular domain
///
/// This is created by `Domain::parse` or `List::parse_domain`.
#[derive(Debug, Clone)]
pub struct Domain {
    full: String,
    typ: Option<Type>,
    suffix: Option<String>,
    registrable: Option<String>,
}

const OFFICIAL_URL: &'static str = "https://publicsuffix.org/list/public_suffix_list.dat";

impl List {
    fn append(&mut self, rule: &str, typ: Type) -> Result<()> {
        rule.rsplit('.').next()
            .ok_or(ErrorKind::InvalidRule(rule.into()).into())
            .and_then(|tld| {
                if tld.is_empty() {
                    return Err(ErrorKind::InvalidRule(rule.into()).into());
                }
                Ok(tld)})
            .and_then(|tld| {
                self.rules.entry(tld.into()).or_insert(Vec::new())
                    .push(Suffix {
                        rule: rule.into(),
                        typ: typ,
                    });
                Ok(())
            })
    }

    fn build<R: Read>(mut reader: R) -> Result<List> {
        let mut res = String::new();
        reader.read_to_string(&mut res)?;

        let mut typ = Type::Icann;

        let mut list = List {
            rules: HashMap::new(),
        };

        for line in res.lines() {
            match line {
                line if line.contains("BEGIN ICANN DOMAINS") => { typ = Type::Icann; }
                line if line.contains("BEGIN PRIVATE DOMAINS") => { typ = Type::Private; }
                line if line.starts_with("//") => { continue; }
                line => {
                    let rule = match line.split_whitespace().next() {
                        Some(rule) => rule,
                        None => continue,
                    };
                    list.append(rule, typ)?;
                }
            }
        }

        Ok(list)
    }

    /// Pull the list from a URL
    pub fn from_url<U: IntoUrl>(url: U) -> Result<List> {
        reqwest::get(url)
            .map_err(|err| ErrorKind::Request(err).into())
            .and_then(Self::build)
    }

    /// Fetch the list from a local file
    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<List> {
        File::open(path)
            .map_err(|err| ErrorKind::Io(err).into())
            .and_then(Self::build)
    }

    /// Pull the list from the official URL
    pub fn fetch() -> Result<List> {
        Self::from_url(OFFICIAL_URL)
    }

    fn find_type(&self, typ: Type) -> Vec<&String> {
        self.rules.values()
            .fold(Vec::new(), |mut res, ref suffices| {
                for suffix in *suffices {
                    if suffix.typ == typ {
                        res.push(&suffix.rule);
                    }
                }
                res
            })
    }

    /// Gets a list of all ICANN domain suffices
    pub fn icann(&self) -> Vec<&String> {
        self.find_type(Type::Icann)
    }

    /// Gets a list of all private domain suffices
    pub fn private(&self) -> Vec<&String> {
        self.find_type(Type::Private)
    }

    /// Gets a list of all domain suffices
    pub fn all(&self) -> Vec<&String> {
        self.rules.values()
            .fold(Vec::new(), |mut res, ref suffices| {
                for suffix in *suffices {
                    res.push(&suffix.rule);
                }
                res
            })
    }

    /// Parses a domain using the list
    pub fn parse_domain(&self, domain: &str) -> Result<Domain> {
        Domain::parse(domain, self)
    }
}

impl Domain {
    /// Check if a domain has valid syntax
    pub fn has_valid_syntax(domain: &str) -> bool {
        // we are explicitly checking for this here before calling `domain_to_ascii`
        // because `domain_to_ascii` strips of leading dots so we won't be able to
        // check for this later
        if domain.starts_with('.') { return false; }
        // let's convert the domain to ascii early on so we can validate
        // internationalised domain names as well
        let domain = match domain_to_ascii(domain) {
            Ok(domain) => domain,
            Err(_) => { return false; },
        };
        // all labels must conform to this pattern
        let pattern = Regex::new(r"^([[:alnum:]]+|([[:alnum:]]+([[:alnum:]-]*[[:alnum:]]+)*))$").unwrap();
        let mut labels: Vec<&str> = domain.split('.').collect();
        // strip of the first dot from a domain to support fully qualified domain names
        if domain.ends_with(".") { labels.pop(); }
        labels.reverse();
        for (i, label) in labels.iter().enumerate() {
            // any label must be 63 characters or less
            if label.chars().count() > 63 { return false; }
            // the tld must not be a number
            if i == 0 && label.parse::<f64>().is_ok() { return false; }
            // any label must only contain allowed characters
            if !pattern.is_match(label) { return false; }
        }
        true
    }

    fn find_possible_matches<'a>(domain: &str, list: &'a List) -> Result<Vec<&'a Suffix>> {
        let tld = match domain.rsplit('.').next() {
            Some(tld) => {
                if tld.is_empty() { return Ok(Vec::new()); }
                tld
            },
            None => { return Ok(Vec::new()); },
        };
        let candidates = match list.rules.get(tld) {
            Some(candidates) => candidates,
            None => { return Ok(Vec::new()); },
        };
        let candidates = candidates.iter()
            .fold(Vec::new(), |mut res, ref suffix| {
                res.push(*suffix);
                res
            });
        Ok(candidates)
    }

    fn assemble(input: &str, s_len: usize) -> String {
        let domain = input.to_lowercase();

        let d_labels: Vec<&str> = domain
            .trim_right_matches('.')
            .split('.').rev().collect();

        (&d_labels[..s_len]).iter().rev()
            .map(|part| *part)
            .collect::<Vec<_>>()
            .join(".")
    }

    fn find_match(input: &str, domain: &str, candidates: Vec<&Suffix>) -> Result<Domain> {
        let d_labels: Vec<&str> = domain.split('.').rev().collect();

        let mut registrable = None;
        let mut suffix = None;
        let mut typ = None;
        let mut num_labels = 0;

        let no_possible_matches_found = candidates.is_empty();

        for candidate in candidates {
            let s_labels: Vec<&str> = candidate.rule.split('.').rev().collect();
            if s_labels.len() > d_labels.len() { continue; }
            for (i, label) in s_labels.iter().enumerate() {
                if *label == d_labels[i] || *label == "*" || label.trim_left_matches('!') == d_labels[i] {
                    if i == s_labels.len()-1 {
                        if s_labels.len() >= num_labels {
                            num_labels = s_labels.len();
                            typ = Some(candidate.typ);
                            let s_len = if label.starts_with("!") {
                                s_labels.len()-1
                            } else {
                                s_labels.len()
                            };
                            suffix = Some(Self::assemble(input, s_len));
                            if d_labels.len() > s_len {
                                registrable = Some(Self::assemble(input, s_len+1));
                            } else {
                                registrable = None;
                            }
                        }
                    }
                } else {
                    break;
                }
            }
        }

        if suffix.is_none() && d_labels.len() > 1 && no_possible_matches_found {
            suffix = Some(Self::assemble(input, 1));
            registrable = Some(Self::assemble(input, 2));
        }

        Ok(Domain {
            full: input.to_string(),
            typ: typ,
            suffix: suffix,
            registrable: registrable,
        })
    }

    /// Parses a domain using the list
    pub fn parse(domain: &str, list: &List) -> Result<Domain> {
        let input = domain;
        if !Self::has_valid_syntax(input) {
            return Err(ErrorKind::InvalidDomain(input.into()).into());
        }
        let domain = input.trim().trim_right_matches('.');
        let (domain, res) = domain_to_unicode(domain);
        if let Err(errors) = res {
            return Err(ErrorKind::Uts46(errors).into());
        }
        Self::find_possible_matches(&domain, list)
            .and_then(|res| Self::find_match(input, &domain, res))
    }

    /// Gets the root domain portion if any
    pub fn root_domain(&self) -> Option<&String> {
        match self.registrable {
            Some(ref registrable) => Some(registrable),
            None => None,
        }
    }

    /// Gets the suffix if any
    pub fn suffix(&self) -> Option<&String> {
        match self.suffix {
            Some(ref suffix) => Some(suffix),
            None => None,
        }
    }

    /// Whether the domain has a private suffix
    pub fn is_private(&self) -> bool {
        match self.typ {
            Some(typ) => match typ {
                Type::Icann => false,
                Type::Private => true,
            },
            None => false,
        }
    }

    /// Whether the domain has an ICANN suffix
    pub fn is_icann(&self) -> bool {
        match self.typ {
            Some(typ) => match typ {
                Type::Icann => true,
                Type::Private => false,
            },
            None => false,
        }
    }

    /// Whether this domain's suffix is in the list
    ///
    /// If it is, this is definately a valid domain. If it's not
    /// chances are very high that this isn't a valid domain name,
    /// however, it might simply be because the suffix is new and
    /// it hasn't been added to the list yet.
    ///
    /// If you want to validate a domain name, use this as a quick
    /// check but fall back to a DNS lookup if it returns false.
    pub fn has_known_suffix(&self) -> bool {
        self.typ.is_some()
    }
}
<commit_msg>Refactor the regex<commit_after>//! The Public Suffix List Client Library
//!
//! This library allows you to easily and accurately validate
//! or break down any given domain name.
//!
//! ## Examples
//!
//! ```
//! extern crate publicsuffix;
//!
//! use publicsuffix::List;
//!
//! fn main() {
//!     let list = List::fetch().unwrap();
//!
//!     let domain = list.parse_domain("www.example.com").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("example.com"));
//!
//!     let domain = list.parse_domain("www.食狮.中国").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("食狮.中国"));
//!
//!     let domain = list.parse_domain("www.xn--85x722f.xn--55qx5d.cn").unwrap();
//!     let tld = domain.root_domain().unwrap();
//!     assert_eq!(tld, &String::from("xn--85x722f.xn--55qx5d.cn"));
//!
//!     let domain = list.parse_domain("a.b.domain.biz").unwrap();
//!     let suffix = domain.suffix().unwrap();
//!     assert_eq!(suffix, &String::from("biz"));
//!
//!     let domain = list.parse_domain("a.b.example.uk.com").unwrap();
//!     let suffix = domain.suffix().unwrap();
//!     assert_eq!(suffix, &String::from("uk.com"));
//! }

#![recursion_limit = "1024"]

#[macro_use]
extern crate error_chain;
extern crate reqwest;
extern crate regex;
extern crate idna;

pub mod errors;

#[cfg(test)]
mod tests;

use std::io::Read;
use std::fs::File;
use std::path::Path;
use std::collections::HashMap;

use errors::*;
use regex::Regex;
use reqwest::IntoUrl;
use idna::{domain_to_unicode, domain_to_ascii};

#[derive(Debug)]
struct Suffix {
    rule: String,
    typ: Type,
}

/// Stores the public suffix list
///
/// You can use the methods, `fetch`, `from_url` or `from_path` to build the list.
/// If you are using this in a long running server it's recommended you use either
/// `fetch` or `from_url` to download updates at least once a week.
#[derive(Debug)]
pub struct List {
    rules: HashMap<String, Vec<Suffix>>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
enum Type {
    Icann,
    Private,
}

/// Holds information about a particular domain
///
/// This is created by `Domain::parse` or `List::parse_domain`.
#[derive(Debug, Clone)]
pub struct Domain {
    full: String,
    typ: Option<Type>,
    suffix: Option<String>,
    registrable: Option<String>,
}

const OFFICIAL_URL: &'static str = "https://publicsuffix.org/list/public_suffix_list.dat";

impl List {
    fn append(&mut self, rule: &str, typ: Type) -> Result<()> {
        rule.rsplit('.').next()
            .ok_or(ErrorKind::InvalidRule(rule.into()).into())
            .and_then(|tld| {
                if tld.is_empty() {
                    return Err(ErrorKind::InvalidRule(rule.into()).into());
                }
                Ok(tld)})
            .and_then(|tld| {
                self.rules.entry(tld.into()).or_insert(Vec::new())
                    .push(Suffix {
                        rule: rule.into(),
                        typ: typ,
                    });
                Ok(())
            })
    }

    fn build<R: Read>(mut reader: R) -> Result<List> {
        let mut res = String::new();
        reader.read_to_string(&mut res)?;

        let mut typ = Type::Icann;

        let mut list = List {
            rules: HashMap::new(),
        };

        for line in res.lines() {
            match line {
                line if line.contains("BEGIN ICANN DOMAINS") => { typ = Type::Icann; }
                line if line.contains("BEGIN PRIVATE DOMAINS") => { typ = Type::Private; }
                line if line.starts_with("//") => { continue; }
                line => {
                    let rule = match line.split_whitespace().next() {
                        Some(rule) => rule,
                        None => continue,
                    };
                    list.append(rule, typ)?;
                }
            }
        }

        Ok(list)
    }

    /// Pull the list from a URL
    pub fn from_url<U: IntoUrl>(url: U) -> Result<List> {
        reqwest::get(url)
            .map_err(|err| ErrorKind::Request(err).into())
            .and_then(Self::build)
    }

    /// Fetch the list from a local file
    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<List> {
        File::open(path)
            .map_err(|err| ErrorKind::Io(err).into())
            .and_then(Self::build)
    }

    /// Pull the list from the official URL
    pub fn fetch() -> Result<List> {
        Self::from_url(OFFICIAL_URL)
    }

    fn find_type(&self, typ: Type) -> Vec<&String> {
        self.rules.values()
            .fold(Vec::new(), |mut res, ref suffices| {
                for suffix in *suffices {
                    if suffix.typ == typ {
                        res.push(&suffix.rule);
                    }
                }
                res
            })
    }

    /// Gets a list of all ICANN domain suffices
    pub fn icann(&self) -> Vec<&String> {
        self.find_type(Type::Icann)
    }

    /// Gets a list of all private domain suffices
    pub fn private(&self) -> Vec<&String> {
        self.find_type(Type::Private)
    }

    /// Gets a list of all domain suffices
    pub fn all(&self) -> Vec<&String> {
        self.rules.values()
            .fold(Vec::new(), |mut res, ref suffices| {
                for suffix in *suffices {
                    res.push(&suffix.rule);
                }
                res
            })
    }

    /// Parses a domain using the list
    pub fn parse_domain(&self, domain: &str) -> Result<Domain> {
        Domain::parse(domain, self)
    }
}

impl Domain {
    /// Check if a domain has valid syntax
    pub fn has_valid_syntax(domain: &str) -> bool {
        // we are explicitly checking for this here before calling `domain_to_ascii`
        // because `domain_to_ascii` strips of leading dots so we won't be able to
        // check for this later
        if domain.starts_with('.') { return false; }
        // let's convert the domain to ascii early on so we can validate
        // internationalised domain names as well
        let domain = match domain_to_ascii(domain) {
            Ok(domain) => domain,
            Err(_) => { return false; },
        };
        // all labels must conform to this pattern
        let pattern = Regex::new(r"^([[:alnum:]]+|([[:alnum:]]+[[:alnum:]-]*[[:alnum:]]+))$").unwrap();
        let mut labels: Vec<&str> = domain.split('.').collect();
        // strip of the first dot from a domain to support fully qualified domain names
        if domain.ends_with(".") { labels.pop(); }
        labels.reverse();
        for (i, label) in labels.iter().enumerate() {
            // any label must be 63 characters or less
            if label.chars().count() > 63 { return false; }
            // the tld must not be a number
            if i == 0 && label.parse::<f64>().is_ok() { return false; }
            // any label must only contain allowed characters
            if !pattern.is_match(label) { return false; }
        }
        true
    }

    fn find_possible_matches<'a>(domain: &str, list: &'a List) -> Result<Vec<&'a Suffix>> {
        let tld = match domain.rsplit('.').next() {
            Some(tld) => {
                if tld.is_empty() { return Ok(Vec::new()); }
                tld
            },
            None => { return Ok(Vec::new()); },
        };
        let candidates = match list.rules.get(tld) {
            Some(candidates) => candidates,
            None => { return Ok(Vec::new()); },
        };
        let candidates = candidates.iter()
            .fold(Vec::new(), |mut res, ref suffix| {
                res.push(*suffix);
                res
            });
        Ok(candidates)
    }

    fn assemble(input: &str, s_len: usize) -> String {
        let domain = input.to_lowercase();

        let d_labels: Vec<&str> = domain
            .trim_right_matches('.')
            .split('.').rev().collect();

        (&d_labels[..s_len]).iter().rev()
            .map(|part| *part)
            .collect::<Vec<_>>()
            .join(".")
    }

    fn find_match(input: &str, domain: &str, candidates: Vec<&Suffix>) -> Result<Domain> {
        let d_labels: Vec<&str> = domain.split('.').rev().collect();

        let mut registrable = None;
        let mut suffix = None;
        let mut typ = None;
        let mut num_labels = 0;

        let no_possible_matches_found = candidates.is_empty();

        for candidate in candidates {
            let s_labels: Vec<&str> = candidate.rule.split('.').rev().collect();
            if s_labels.len() > d_labels.len() { continue; }
            for (i, label) in s_labels.iter().enumerate() {
                if *label == d_labels[i] || *label == "*" || label.trim_left_matches('!') == d_labels[i] {
                    if i == s_labels.len()-1 {
                        if s_labels.len() >= num_labels {
                            num_labels = s_labels.len();
                            typ = Some(candidate.typ);
                            let s_len = if label.starts_with("!") {
                                s_labels.len()-1
                            } else {
                                s_labels.len()
                            };
                            suffix = Some(Self::assemble(input, s_len));
                            if d_labels.len() > s_len {
                                registrable = Some(Self::assemble(input, s_len+1));
                            } else {
                                registrable = None;
                            }
                        }
                    }
                } else {
                    break;
                }
            }
        }

        if suffix.is_none() && d_labels.len() > 1 && no_possible_matches_found {
            suffix = Some(Self::assemble(input, 1));
            registrable = Some(Self::assemble(input, 2));
        }

        Ok(Domain {
            full: input.to_string(),
            typ: typ,
            suffix: suffix,
            registrable: registrable,
        })
    }

    /// Parses a domain using the list
    pub fn parse(domain: &str, list: &List) -> Result<Domain> {
        let input = domain;
        if !Self::has_valid_syntax(input) {
            return Err(ErrorKind::InvalidDomain(input.into()).into());
        }
        let domain = input.trim().trim_right_matches('.');
        let (domain, res) = domain_to_unicode(domain);
        if let Err(errors) = res {
            return Err(ErrorKind::Uts46(errors).into());
        }
        Self::find_possible_matches(&domain, list)
            .and_then(|res| Self::find_match(input, &domain, res))
    }

    /// Gets the root domain portion if any
    pub fn root_domain(&self) -> Option<&String> {
        match self.registrable {
            Some(ref registrable) => Some(registrable),
            None => None,
        }
    }

    /// Gets the suffix if any
    pub fn suffix(&self) -> Option<&String> {
        match self.suffix {
            Some(ref suffix) => Some(suffix),
            None => None,
        }
    }

    /// Whether the domain has a private suffix
    pub fn is_private(&self) -> bool {
        match self.typ {
            Some(typ) => match typ {
                Type::Icann => false,
                Type::Private => true,
            },
            None => false,
        }
    }

    /// Whether the domain has an ICANN suffix
    pub fn is_icann(&self) -> bool {
        match self.typ {
            Some(typ) => match typ {
                Type::Icann => true,
                Type::Private => false,
            },
            None => false,
        }
    }

    /// Whether this domain's suffix is in the list
    ///
    /// If it is, this is definately a valid domain. If it's not
    /// chances are very high that this isn't a valid domain name,
    /// however, it might simply be because the suffix is new and
    /// it hasn't been added to the list yet.
    ///
    /// If you want to validate a domain name, use this as a quick
    /// check but fall back to a DNS lookup if it returns false.
    pub fn has_known_suffix(&self) -> bool {
        self.typ.is_some()
    }
}
<|endoftext|>


<commit_before>//! Promises in Rust.

#![warn(incomplete_docs)]

use std::thread;
use std::sync::mpsc::channel;
use std::time::Duration;
use std::thread::JoinHandle;
use std::marker::{Send};
use std::sync::mpsc::{Sender, Receiver, TryRecvError};

/// A promise is a way of doing work in the background
pub struct Promise<T, E> where T: Send, E: Send {
    //sender: Sender<PromiseMessage<T, E>>, // Send messages to the promise
    receiver: Receiver<Result<T, E>> // Get the result of the promise for chaining (pass into then)
}

enum PromiseJoinType {
    Race,
    All
}

impl<T, E> Promise<T, E> where T: Send, E: Send {

    /// Chains a function to be called after this promise resolves.
    pub fn then<T2, E2>(self, callback: fn(f: T) -> Result<T2, E2>,
                              errback:  fn(e: E) -> Result<T2, E2>)
                        -> Promise<T2, E2>
    where T2: Send, E2: Send {
        let recv = self.receiver;
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            if let Ok(message) = recv.recv() { // Blocking receive until message
                match message {
                    Ok(val) => {
                        let _ = tx.send(callback(val)).unwrap_or(());
                    }
                    Err(err) => {
                        let _ = tx.send(errback(err)).unwrap_or(());
                    }
                }
            }
            else { // Receive failed, origin was dropped
            }
        });
        return Promise { receiver: rx };
    }

    /// Chains a function to be called after this promise resolves.
    pub fn then_result<T2, E2>(self,
                               callback: fn(input: Result<T, E>) -> Result<T2, E2>)
                               -> Promise<T2, E2>
    where T2: Sync, E2: Sync {
        let recv = self.receiver;
        let recv2 = self.receiver;
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            if let Ok(result) = recv.recv() { // Blocking receive until message
                let _ = tx.send(callback(result)).unwrap_or(());
            }
            else { // Receive failed, origin was dropped
            }
        });
        return Promise { receiver: rx };
    }

    /// Creates a new promsie
    pub fn new(func: fn() -> Result<T, E>) -> Promise<T, E> {
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            let result = func();
            let _ = tx.send(result).unwrap_or(());
        });

        return Promise { receiver: rx };
    }
}

pub fn race<T, E>(promises: Vec<Promise<T, E>>) -> Promise<T, E>
where T: Send, E: Send {
    let mut receivers = promises.into_iter().map(|x| x.receiver).collect();
    let (tx, rx) = channel();

    thread::spawn(move || {
        'outer: loop {
            for i in 0..receivers.len() {
                match receivers[i].try_recv() {
                    Ok(val) => {
                        let _ = tx.send(val).unwrap_or(());
                        return;
                    }
                    Err(err) => {
                        if err == TryRecvError::Disconnected {
                            receivers.remove(i);
                        }
                    }
                }
            }
        }
    });
}

pub fn all<T, E>(promises: Vec<Promise<T, E>>) -> Promise<T, E> {
}

pub fn resolve<T, E>(val: E) -> Promise<T, E> {
}

pub fn reject<T, E>(val: E) -> Promise<T, E> {
}

#[test]
fn test_channels() {
    println!("Begin test...");
    let mut guard: Option<JoinHandle<_>> = None;
    let mut rec: Option<Receiver<_>> = None;

    {
        println!("\tBegin inner scope...");
        let (tx, rx) = channel();
        guard = Some(thread::spawn(move || {
            println!("\t\tThread created!");
            tx.send("Hello!").unwrap();
            println!("\t\tThread finished!");
        }));
        rec = Some(rx);
        println!("\t(tx, rx) out of scope");
    }

    println!("Sleeping...");
    thread::sleep(Duration::from_secs(2));
    println!("Receiving result...");
    let result = rec.unwrap().recv();
    assert_eq!(result, Ok("Hello!"));
}
<commit_msg>Clean up race and all<commit_after>//! Promises in Rust.

#![warn(missing_docs)]

use std::thread;
use std::sync::mpsc::channel;
use std::time::Duration;
use std::thread::JoinHandle;
use std::marker::{Send};
use std::sync::mpsc::{Sender, Receiver, TryRecvError};

type Promiseable = Send + 'static;

/// A promise is a way of doing work in the background
pub struct Promise<T, E> where T: Send, E: Send {
    receiver: Receiver<Result<T, E>> // Get the result of the promise for chaining (pass into then)
}

impl<T, E> Promise<T, E> where T: Send + 'static, E: Send + 'static {

    /// Chains a function to be called after this promise resolves.
    pub fn then<T2, E2>(self, callback: fn(f: T) -> Result<T2, E2>,
                              errback:  fn(e: E) -> Result<T2, E2>)
                        -> Promise<T2, E2>
    where T2: Send + 'static, E2: Send + 'static {
        let recv = self.receiver;
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            if let Ok(message) = recv.recv() { // Blocking receive until message
                match message {
                    Ok(val) => {
                        let _ = tx.send(callback(val)).unwrap_or(());
                    }
                    Err(err) => {
                        let _ = tx.send(errback(err)).unwrap_or(());
                    }
                }
            }
            else { // Receive failed, origin was dropped
            }
        });
        return Promise { receiver: rx };
    }

    /// Chains a function to be called after this promise resolves.
    pub fn then_result<T2, E2>(self,
                               callback: fn(input: Result<T, E>) -> Result<T2, E2>)
                               -> Promise<T2, E2>
    where T2: Send + 'static, E2: Send + 'static {
        let recv = self.receiver;
        let recv2 = self.receiver;
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            if let Ok(result) = recv.recv() { // Blocking receive until message
                let _ = tx.send(callback(result)).unwrap_or(());
            }
            else { // Receive failed, origin was dropped
            }
        });
        return Promise { receiver: rx };
    }

    /// Creates a new promsie
    pub fn new<F>(func: F) -> Promise<T, E>
        where T: Send + 'static, E: Send + 'static,
              F: Send + 'static + FnOnce() -> Result<T, E> {
        let (tx, rx) = channel();

        let thread = thread::spawn(move || {
            let result = func();
            let _ = tx.send(result).unwrap_or(());
        });

        return Promise { receiver: rx };
    }
}

pub fn race<T, E>(promises: Vec<Promise<T, E>>) -> Promise<T, E>
where T: Send + 'static, E: Send + 'static {
    let mut receivers = promises.into_iter().map(|p| p.receiver).collect();
    let (tx, rx) = channel();

    thread::spawn(move || {
        race_function(receivers, tx);
    });

    return Promise { receiver: rx };
}

pub fn all<T, E>(promises: Vec<Promise<T, E>>) -> Promise<Vec<T>, E>
where T: Send + 'static, E: Send + 'static {
    let receivers: Vec<Receiver<Result<T, E>>> = promises.into_iter().map(|p| p.receiver).collect();
    let (tx, rx) = channel();

    thread::spawn(move || {
        all_function(receivers, tx);
    });

    return Promise { receiver: rx };
}

pub fn resolve<T, E>(val: E) -> Promise<T, E> {
}

pub fn reject<T, E>(val: E) -> Promise<T, E> {
}

fn race_function<T, E>(receivers: Vec<Receiver<Result<T, E>>>, tx: Sender<Result<T, E>>)
where T: Send + 'static, E: Send + 'static {
    'outer: loop {
        for i in 0..receivers.len() {
            match receivers[i].try_recv() {
                Ok(val) => {
                    let _ = tx.send(val).unwrap_or(());
                    return;
                }
                Err(err) => {
                    if err == TryRecvError::Disconnected {
                        receivers.remove(i);
                    }
                }
            }
        }
    }
}

fn all_function<T, E>(receivers: Vec<Receiver<Result<T, E>>>, tx: Sender<Result<Vec<T>, E>>)
where T: Send + 'static, E: Send + 'static {
    let mut values: Vec<T> = Vec::with_capacity(receivers.len());
    let mut mut_receivers = receivers; //.into_iter().map(|r| r.clone()).collect();
    'outer: loop {
        for rec in receivers {
            match rec.try_recv() {
                Ok(val) => {
                    match val {
                        Ok(t) => values.push(t),
                        Err(e) => {
                            let _ = tx.send(Err(e)).unwrap_or(());
                            return;
                        }
                    }
                }
                Err(err) => {
                    // Remove dead promises?
                }
            }
        }
    }
}

#[test]
fn test_channels() {
    println!("Begin test...");
    let mut guard: Option<JoinHandle<_>> = None;
    let mut rec: Option<Receiver<_>> = None;

    {
        println!("\tBegin inner scope...");
        let (tx, rx) = channel();
        guard = Some(thread::spawn(move || {
            println!("\t\tThread created!");
            tx.send("Hello!").unwrap();
            println!("\t\tThread finished!");
        }));
        rec = Some(rx);
        println!("\t(tx, rx) out of scope");
    }

    println!("Sleeping...");
    thread::sleep(Duration::from_secs(2));
    println!("Receiving result...");
    let result = rec.unwrap().recv();
    assert_eq!(result, Ok("Hello!"));
}
<|endoftext|>


<commit_before><commit_msg>tab -> spaces<commit_after><|endoftext|>


<commit_before>#![feature(vec_push_all)]

extern crate unicode_segmentation;
use unicode_segmentation::UnicodeSegmentation;

use std::rc::Rc;

// const SPLIT_LEN: usize = 10;
// const JOIN_LEN: usize = 5;
// const REBALANCE_RATIO: f32 = 1.2;

pub enum Rope {
    Leaf { base: Rc<String>,
           start: usize,
           end: usize,
           graphemes: usize },
    Concat { left: Rc<Rope>,
             right: Rc<Rope>,
             graphemes: usize },
}
impl Rope {
    /// Create a new rope leaf
    pub fn from_str(base: &str) -> Rope {
        let num_graphemes = count_grapheme_clusters(base);
        Rope::Leaf { base: Rc::new(String::from(base)),
                     start: 0,
                     end: base.len(),
                     graphemes: num_graphemes }
    }
    /// Gets the number of Unicode grapheme clusters in the children of this node
    pub fn num_graphemes(&self) -> usize {
        match *self {
            Rope::Leaf { graphemes, .. } => graphemes,
            Rope::Concat { graphemes, .. } => graphemes,
        }
    }
    /// Split a Rope after the nth grapheme
    pub fn split(&self, grapheme_index: usize) -> Option<(Rope, Rope)> {
        match *self {
            Rope::Leaf { ref base, start, end, .. } => {
                grapheme_byte_index(&base[start..end], grapheme_index)
                    .map(|byte_index| {
                        let left = Rope::Leaf { base: base.clone(),
                                                start: start,
                                                end: start+byte_index,
                                                graphemes: count_grapheme_clusters(&base[start..start+byte_index]) };
                        let right = Rope::Leaf { base: base.clone(),
                                                 start: start+byte_index,
                                                 end: end,
                                                 graphemes: count_grapheme_clusters(&base[start+byte_index..end]) };
                        (left, right)
                    })
            },
            Rope::Concat{ ref left, ref right, .. } => {
                // self is (left, right)
                if left.num_graphemes() >= grapheme_index {
                    // split left into (l1, l2), return (l1, concat(l2, right))
                    left.split(grapheme_index).map(|(l1, l2)| {
                        (l1, Rope::Concat { left: Rc::new(l2.clone()),
                                            right: right.clone(),
                                            graphemes: l2.num_graphemes() + right.num_graphemes() })
                    })
                } else {
                    // split right into (r1, r2), return (concat(left, r1), r2)
                    right.split(grapheme_index - left.num_graphemes()).map(|(r1, r2)| {
                        (Rope::Concat { left: left.clone(),
                                        right: Rc::new(r1.clone()),
                                        graphemes: left.num_graphemes() + r1.num_graphemes() },
                         r2)
                    })
                }
            }
        }
    }
    /// Collect the pieces of this Rope into a continuous String
    pub fn to_string(&self) -> String {
        let mut buf = String::with_capacity(self.num_graphemes());

        fn str_parts<'a>(root: &'a Rope) -> Vec<&'a str> {
            let mut v = vec![];
            match *root {
                Rope::Leaf { ref base, start, end, .. } => {
                    v.push( &base[start..end] );
                },
                Rope::Concat { ref left, ref right, .. } => {
                    v.push_all(&str_parts(&left));
                    v.push_all(&str_parts(&right));
                },
            }
            v
        }
        for part in str_parts(self).iter() {
            buf.push_str(part);
        }

        buf
    }
    /// Join two pieces into a Rope
    pub fn join(left: Rope, right: Rope) -> Rope {
        let graphemes = left.num_graphemes() + right.num_graphemes();
        Rope::Concat { left: Rc::new(left),
                       right: Rc::new(right),
                       graphemes: graphemes }
    }
    /// Insert a string into a Rope by splitting and joining nodes
    /// Returns a new Rope, leaving the original unchanged
    pub fn insert(&self, grapheme_index: usize, value: &str) -> Option<Rope> {
        self.split(grapheme_index).map(|(left, right)| {
            let left = Rope::join(left, Rope::from_str(value));
            Rope::join(left, right)
        })
    }
}
impl Clone for Rope {
    fn clone(&self) -> Rope {
        match *self {
            Rope::Leaf { ref base, start, end, graphemes } => Rope::Leaf { base: base.clone(),
                                                                           start: start,
                                                                           end: end,
                                                                           graphemes: graphemes },
            Rope::Concat { ref left, ref right, graphemes } => Rope::Concat { left: left.clone(),
                                                                              right: right.clone(),
                                                                              graphemes: graphemes },
        }
    }
}
/// Count the number of unicode extended grapheme clusters in a string
/// This is O(n)
fn count_grapheme_clusters(s: &str) -> usize {
    UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>().len()
}

/// Find the byte index of the nth grapheme cluster in a string
/// Returns None if the grapheme index is out of bounds
fn grapheme_byte_index(base: &str, nth_grapheme: usize) -> Option<usize> {
    UnicodeSegmentation::grapheme_indices(base, true)
        .nth(nth_grapheme).map(|(i, _)| i)
}

#[test]
fn test_count_grapheme_clusters() {
    // 11 bytes, 3 grapheme clusters
    let s = "a̐éö̲";
    assert_eq!(s.len(), 11);
    assert_eq!(count_grapheme_clusters(s), 3);
}

#[test]
fn test_create_leaf() {
    let leaf = Rope::from_str("a̐éö̲");
    assert_eq!(leaf.num_graphemes(), 3);
}

#[test]
fn test_split_leaf() {
    let leaf = Rope::from_str("a̐éö̲a̐éö̲");
    let (left, right) = leaf.split(3).unwrap();
    assert_eq!(left.num_graphemes(), right.num_graphemes());
    assert_eq!(left.to_string(), "a̐éö̲");
}

#[test]
fn test_create_rope() {
    let rope = Rope::from_str("The quick brown fox jumps over the lazy dog.");
    assert_eq!(rope.num_graphemes(), 44);
    assert_eq!(rope.to_string(), "The quick brown fox jumps over the lazy dog.");
}

#[test]
fn test_rope_operations() {
    let rope = Rope::from_str("The quick brown fox jumps over the lazy dog.");

    let (left, right) = rope.split(22).unwrap();
    assert_eq!(left.to_string(), "The quick brown fox ju");
    assert_eq!(right.to_string(), "mps over the lazy dog.");

    let rope = Rope::join(left, Rope::from_str("mps over the tiny wooden fence!"));
    assert_eq!(rope.to_string(), "The quick brown fox jumps over the tiny wooden fence!");

    let rope2 = rope.insert(10, "(and really very clever) ").unwrap();
    assert_eq!(rope2.to_string(), "The quick (and really very clever) brown fox jumps over the tiny wooden fence!");
}
<commit_msg>Added append function<commit_after>#![feature(vec_push_all)]

extern crate unicode_segmentation;
use unicode_segmentation::UnicodeSegmentation;

use std::rc::Rc;

// const SPLIT_LEN: usize = 10;
// const JOIN_LEN: usize = 5;
// const REBALANCE_RATIO: f32 = 1.2;

pub enum Rope {
    Leaf { base: Rc<String>,
           start: usize,
           end: usize,
           graphemes: usize },
    Concat { left: Rc<Rope>,
             right: Rc<Rope>,
             graphemes: usize },
}
impl Rope {
    /// Create a new rope leaf
    pub fn from_str(base: &str) -> Rope {
        let num_graphemes = count_grapheme_clusters(base);
        Rope::Leaf { base: Rc::new(String::from(base)),
                     start: 0,
                     end: base.len(),
                     graphemes: num_graphemes }
    }
    /// Gets the number of Unicode grapheme clusters in the children of this node
    pub fn num_graphemes(&self) -> usize {
        match *self {
            Rope::Leaf { graphemes, .. } => graphemes,
            Rope::Concat { graphemes, .. } => graphemes,
        }
    }
    /// Split a Rope after the nth grapheme
    pub fn split(&self, grapheme_index: usize) -> Option<(Rope, Rope)> {
        match *self {
            Rope::Leaf { ref base, start, end, .. } => {
                grapheme_byte_index(&base[start..end], grapheme_index)
                    .map(|byte_index| {
                        let left = Rope::Leaf { base: base.clone(),
                                                start: start,
                                                end: start+byte_index,
                                                graphemes: count_grapheme_clusters(&base[start..start+byte_index]) };
                        let right = Rope::Leaf { base: base.clone(),
                                                 start: start+byte_index,
                                                 end: end,
                                                 graphemes: count_grapheme_clusters(&base[start+byte_index..end]) };
                        (left, right)
                    })
            },
            Rope::Concat{ ref left, ref right, .. } => {
                // self is (left, right)
                if left.num_graphemes() >= grapheme_index {
                    // split left into (l1, l2), return (l1, concat(l2, right))
                    left.split(grapheme_index).map(|(l1, l2)| {
                        (l1, Rope::Concat { left: Rc::new(l2.clone()),
                                            right: right.clone(),
                                            graphemes: l2.num_graphemes() + right.num_graphemes() })
                    })
                } else {
                    // split right into (r1, r2), return (concat(left, r1), r2)
                    right.split(grapheme_index - left.num_graphemes()).map(|(r1, r2)| {
                        (Rope::Concat { left: left.clone(),
                                        right: Rc::new(r1.clone()),
                                        graphemes: left.num_graphemes() + r1.num_graphemes() },
                         r2)
                    })
                }
            }
        }
    }
    /// Collect the pieces of this Rope into a continuous String
    pub fn to_string(&self) -> String {
        let mut buf = String::with_capacity(self.num_graphemes());

        fn str_parts<'a>(root: &'a Rope) -> Vec<&'a str> {
            let mut v = vec![];
            match *root {
                Rope::Leaf { ref base, start, end, .. } => {
                    v.push( &base[start..end] );
                },
                Rope::Concat { ref left, ref right, .. } => {
                    v.push_all(&str_parts(&left));
                    v.push_all(&str_parts(&right));
                },
            }
            v
        }
        for part in str_parts(self).iter() {
            buf.push_str(part);
        }

        buf
    }
    /// Join two pieces into a Rope
    pub fn join(left: Rope, right: Rope) -> Rope {
        let graphemes = left.num_graphemes() + right.num_graphemes();
        Rope::Concat { left: Rc::new(left),
                       right: Rc::new(right),
                       graphemes: graphemes }
    }
    /// Insert a string into a Rope by splitting and joining nodes
    /// Returns a new Rope, leaving the original unchanged
    pub fn insert(&self, grapheme_index: usize, value: &str) -> Option<Rope> {
        self.split(grapheme_index).map(|(left, right)| {
            let left = Rope::join(left, Rope::from_str(value));
            Rope::join(left, right)
        })
    }
    /// Append a string to the Rope, returning a new instance and leaving the original unchanged
    pub fn append(&self, value: &str) -> Rope {
        Rope::join(self.clone(), Rope::from_str(value))
    }
}
impl Clone for Rope {
    fn clone(&self) -> Rope {
        match *self {
            Rope::Leaf { ref base, start, end, graphemes } => Rope::Leaf { base: base.clone(),
                                                                           start: start,
                                                                           end: end,
                                                                           graphemes: graphemes },
            Rope::Concat { ref left, ref right, graphemes } => Rope::Concat { left: left.clone(),
                                                                              right: right.clone(),
                                                                              graphemes: graphemes },
        }
    }
}
/// Count the number of unicode extended grapheme clusters in a string
/// This is O(n)
fn count_grapheme_clusters(s: &str) -> usize {
    UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>().len()
}

/// Find the byte index of the nth grapheme cluster in a string
/// Returns None if the grapheme index is out of bounds
fn grapheme_byte_index(base: &str, nth_grapheme: usize) -> Option<usize> {
    UnicodeSegmentation::grapheme_indices(base, true)
        .nth(nth_grapheme).map(|(i, _)| i)
}

#[test]
fn test_count_grapheme_clusters() {
    // 11 bytes, 3 grapheme clusters
    let s = "a̐éö̲";
    assert_eq!(s.len(), 11);
    assert_eq!(count_grapheme_clusters(s), 3);
}

#[test]
fn test_create_leaf() {
    let leaf = Rope::from_str("a̐éö̲");
    assert_eq!(leaf.num_graphemes(), 3);
}

#[test]
fn test_split_leaf() {
    let leaf = Rope::from_str("a̐éö̲a̐éö̲");
    let (left, right) = leaf.split(3).unwrap();
    assert_eq!(left.num_graphemes(), right.num_graphemes());
    assert_eq!(left.to_string(), "a̐éö̲");
}

#[test]
fn test_create_rope() {
    let rope = Rope::from_str("The quick brown fox jumps over the lazy dog.");
    assert_eq!(rope.num_graphemes(), 44);
    assert_eq!(rope.to_string(), "The quick brown fox jumps over the lazy dog.");
}

#[test]
fn test_rope_operations() {
    let rope = Rope::from_str("The quick brown fox jumps over the lazy dog.");

    let (left, right) = rope.split(22).unwrap();
    assert_eq!(left.to_string(), "The quick brown fox ju");
    assert_eq!(right.to_string(), "mps over the lazy dog.");

    let rope = Rope::join(left, Rope::from_str("mps over the tiny wooden fence!"));
    assert_eq!(rope.to_string(), "The quick brown fox jumps over the tiny wooden fence!");

    let rope2 = rope.insert(10, "(and really very clever) ").unwrap();
    assert_eq!(rope2.to_string(), "The quick (and really very clever) brown fox jumps over the tiny wooden fence!");

    let rope = rope.append(" One fish two fish, red fish blue fish.");
    assert_eq!(rope.to_string(), "The quick brown fox jumps over the tiny wooden fence! One fish two fish, red fish blue fish.");
}
<|endoftext|>


<commit_before>// This file was generated automatically by crowbook-localize.
// It is probably not a good idea to edit it manually.
//
// # Usage:
//
// ```rust, no_run
// extern crate crowbook_intl_runtime;
// #[macro_use] mod localize_macros;
// crowbook_intl_runtime::set_lang("en");
// lformat!("Hello, {}", name);
// set_lang("fr");
// lformat!("Hello, {}", name);
// ```

use crowbook_intl_runtime;

use std::sync::RwLockReadGuard;

#[doc(hidden)]
pub fn __get_lang() -> RwLockReadGuard<'static, String> {
    crowbook_intl_runtime::__get_lang()
}

<commit_msg>Remove unnecessary import and function definition<commit_after>// This file was generated automatically by crowbook-localize.
// It is probably not a good idea to edit it manually.
//
// # Usage:
//
// ```rust, no_run
// extern crate crowbook_intl_runtime;
// #[macro_use] mod localize_macros;
// crowbook_intl_runtime::set_lang("en");
// lformat!("Hello, {}", name);
// set_lang("fr");
// lformat!("Hello, {}", name);
// ```

<|endoftext|>


<commit_before>use std::{
    collections::{
        hash_map::{Entry, RandomState},
        HashMap,
    },
    hash::{BuildHasher, Hash, Hasher},
    sync::Arc,
    time::SystemTime,
};

use axum::http::StatusCode;
use clap::Clap;
use futures_util::StreamExt;
use rustc_hash::FxHashMap;
use shakmaty::{
    uci::Uci, variant::VariantPosition, zobrist::Zobrist, ByColor, CastlingMode, Outcome, Position,
};
use tokio::{
    sync::{
        mpsc::{self, error::TrySendError},
        watch, RwLock,
    },
    task::JoinHandle,
};

use crate::{
    db::Database,
    model::{
        GameInfo, GameInfoPlayer, Mode, Month, PersonalEntry, PersonalKeyBuilder, PersonalStatus,
        UserId,
    },
};

mod lila;

use lila::{Game, Lila};

#[derive(Clap, Clone)]
pub struct IndexerOpt {
    #[clap(long = "lila", default_value = "https://lichess.org")]
    lila: String,
    #[clap(long = "bearer")]
    bearer: Option<String>,
    #[clap(long = "indexers", default_value = "16")]
    indexers: usize,
}

#[derive(Clone)]
pub struct IndexerStub {
    db: Arc<Database>,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    random_state: RandomState,
    txs: Vec<mpsc::Sender<IndexerMessage>>,
}

impl IndexerStub {
    pub fn spawn(db: Arc<Database>, opt: IndexerOpt) -> (IndexerStub, Vec<JoinHandle<()>>) {
        let random_state = RandomState::new();
        let indexing = Arc::new(RwLock::new(HashMap::new()));
        let mut txs = Vec::with_capacity(opt.indexers);
        let mut join_handles = Vec::with_capacity(opt.indexers);
        for idx in 0..opt.indexers {
            let (tx, rx) = mpsc::channel(500);
            txs.push(tx);
            join_handles.push(tokio::spawn(
                IndexerActor {
                    idx,
                    rx,
                    indexing: Arc::clone(&indexing),
                    db: Arc::clone(&db),
                    lila: Lila::new(opt.clone()),
                }
                .run(),
            ));
        }
        (
            IndexerStub {
                db,
                random_state,
                indexing,
                txs,
            },
            join_handles,
        )
    }

    pub async fn index_player(&self, player: &UserId) -> Option<watch::Receiver<()>> {
        // Optimization: First try subscribing to an existing indexing run,
        // without acquiring a write lock.
        {
            let guard = self.indexing.read().await;
            if let Some(sender) = guard.get(player) {
                return Some(sender.subscribe());
            }
        }

        // Check player indexing status.
        let mut status = self
            .db
            .queryable()
            .get_player_status(player)
            .expect("get player status")
            .unwrap_or_default();

        let since_created_at = match status
            .maybe_revisit_ongoing()
            .or_else(|| status.maybe_index())
        {
            Some(since) => since,
            None => return None, // Do not reindex so soon!
        };

        // Queue indexing request.
        let responsible_indexer = {
            let mut hasher = self.random_state.build_hasher();
            player.hash(&mut hasher);
            hasher.finish() as usize % self.txs.len()
        };

        let mut guard = self.indexing.write().await;
        let entry = match guard.entry(player.to_owned()) {
            Entry::Occupied(entry) => return Some(entry.get().subscribe()),
            Entry::Vacant(entry) => entry,
        };

        match self.txs[responsible_indexer].try_send(IndexerMessage::IndexPlayer {
            player: player.to_owned(),
            status,
            since_created_at,
        }) {
            Ok(_) => {
                let (sender, receiver) = watch::channel(());
                entry.insert(sender);
                Some(receiver)
            }
            Err(TrySendError::Full(_)) => {
                log::error!(
                    "indexer {}: not queuing {} because indexer queue is full",
                    responsible_indexer,
                    player.as_str()
                );
                None
            }
            Err(TrySendError::Closed(_)) => panic!("indexer {} died", responsible_indexer),
        }
    }
}

struct IndexerActor {
    idx: usize,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    rx: mpsc::Receiver<IndexerMessage>,
    db: Arc<Database>,
    lila: Lila,
}

impl IndexerActor {
    async fn run(mut self) {
        while let Some(msg) = self.rx.recv().await {
            match msg {
                IndexerMessage::IndexPlayer {
                    player,
                    status,
                    since_created_at,
                } => {
                    self.index_player(&player, status, since_created_at).await;

                    let mut guard = self.indexing.write().await;
                    guard.remove(&player);
                }
            }
        }
    }

    async fn index_player(
        &self,
        player: &UserId,
        mut status: PersonalStatus,
        since_created_at: u64,
    ) {
        log::info!(
            "indexer {} starting {} (created_at >= {})",
            self.idx,
            player.as_str(),
            since_created_at
        );
        let mut games = match self.lila.user_games(player, since_created_at).await {
            Ok(games) => games,
            Err(err) if err.status() == Some(StatusCode::NOT_FOUND) => {
                log::warn!("indexer did not find player {}", player.as_str());
                return;
            }
            Err(err) => {
                log::error!("indexer {}: request failed: {}", self.idx, err);
                return;
            }
        };

        let hash = ByColor::new_with(|color| PersonalKeyBuilder::with_user_pov(&player, color));
        let mut num_games = 0;
        while let Some(game) = games.next().await {
            let game = match game {
                Ok(game) => game,
                Err(err) => {
                    log::error!("{}", err);
                    continue;
                }
            };

            self.index_game(player, &hash, game, &mut status);

            num_games += 1;
            if num_games % 1024 == 0 {
                log::info!(
                    "indexer {}: indexed {} games for {}",
                    self.idx,
                    num_games,
                    player.as_str()
                );
            }
        }

        status.indexed_at = SystemTime::now();
        self.db
            .queryable()
            .put_player_status(player, status)
            .expect("put player status");
        log::info!(
            "indexer {}: finished indexing {} games for {}",
            self.idx,
            num_games,
            player.as_str()
        );
    }

    fn index_game(
        &self,
        player: &UserId,
        hash: &ByColor<PersonalKeyBuilder>,
        game: Game,
        status: &mut PersonalStatus,
    ) {
        status.latest_created_at = game.created_at;

        if game.status.is_ongoing() {
            if status.revisit_ongoing_created_at.is_none() {
                log::debug!("will revisit ongoing game {} eventually", game.id);
                status.revisit_ongoing_created_at = Some(game.created_at);
            }
            return;
        }

        if game.status.is_unindexable() {
            log::debug!("not indexing {} with status {:?}", game.id, game.status);
            return;
        }

        if game.players.any(|p| p.user.is_none()) {
            return;
        }

        let color = match game
            .players
            .find(|p| p.user.as_ref().map_or(false, |user| user.name == *player))
        {
            Some(color) => color,
            None => {
                log::error!("{} did not play in {}", player.as_str(), game.id);
                return;
            }
        };

        let month = Month::from_time_saturating(game.last_move_at);
        let outcome = Outcome::from_winner(game.winner);

        let queryable = self.db.queryable();
        if queryable
            .get_game_info(game.id)
            .expect("get game info")
            .map_or(false, |info| info.indexed.into_color(color))
        {
            log::debug!("{}/{} already indexed", game.id, color,);
            return;
        }

        let mut batch = queryable.batch();
        batch.merge_game_info(
            game.id,
            GameInfo {
                winner: outcome.winner(),
                speed: game.speed,
                rated: game.rated,
                month,
                players: game.players.map(|p| GameInfoPlayer {
                    name: p.user.map(|p| p.name.to_string()),
                    rating: p.rating,
                }),
                indexed: ByColor::new_with(|c| color == c),
            },
        );

        let variant = game.variant.into();
        let pos = match game.initial_fen {
            Some(fen) => VariantPosition::from_setup(variant, &fen, CastlingMode::Chess960),
            None => Ok(VariantPosition::new(variant)),
        };

        let mut pos: Zobrist<_, u128> = match pos {
            Ok(pos) => Zobrist::new(pos),
            Err(err) => {
                log::warn!("not indexing {}: {}", game.id, err);
                return;
            }
        };

        // Build an intermediate table to remove loops (due to repetitions).
        let mut table: FxHashMap<u128, Uci> =
            FxHashMap::with_capacity_and_hasher(game.moves.len(), Default::default());

        for (ply, san) in game.moves.into_iter().enumerate() {
            let m = match san.to_move(&pos) {
                Ok(m) => m,
                Err(err) => {
                    log::warn!("cutting off {} at ply {}: {}: {}", game.id, ply, err, san);
                    break;
                }
            };

            let uci = m.to_uci(CastlingMode::Chess960);
            table.insert(pos.zobrist_hash(), uci);

            pos.play_unchecked(&m);
        }

        for (zobrist, uci) in table {
            batch.merge_personal(
                hash.by_color(color)
                    .with_zobrist(variant, zobrist)
                    .with_month(month),
                PersonalEntry::new_single(
                    uci.clone(),
                    game.speed,
                    Mode::from_rated(game.rated),
                    game.id,
                    outcome,
                ),
            );
        }

        batch.write().expect("atomically commit game");
    }
}

enum IndexerMessage {
    IndexPlayer {
        player: UserId,
        status: PersonalStatus,
        since_created_at: u64,
    },
}
<commit_msg>some comments and tweaks regarding db consistency<commit_after>use std::{
    collections::{
        hash_map::{Entry, RandomState},
        HashMap,
    },
    hash::{BuildHasher, Hash, Hasher},
    sync::Arc,
    time::SystemTime,
};

use axum::http::StatusCode;
use clap::Clap;
use futures_util::StreamExt;
use rustc_hash::FxHashMap;
use shakmaty::{
    uci::Uci, variant::VariantPosition, zobrist::Zobrist, ByColor, CastlingMode, Outcome, Position,
};
use tokio::{
    sync::{
        mpsc::{self, error::TrySendError},
        watch, RwLock,
    },
    task::JoinHandle,
};

use crate::{
    db::Database,
    model::{
        GameInfo, GameInfoPlayer, Mode, Month, PersonalEntry, PersonalKeyBuilder, PersonalStatus,
        UserId,
    },
};

mod lila;

use lila::{Game, Lila};

#[derive(Clap, Clone)]
pub struct IndexerOpt {
    #[clap(long = "lila", default_value = "https://lichess.org")]
    lila: String,
    #[clap(long = "bearer")]
    bearer: Option<String>,
    #[clap(long = "indexers", default_value = "16")]
    indexers: usize,
}

#[derive(Clone)]
pub struct IndexerStub {
    db: Arc<Database>,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    random_state: RandomState,
    txs: Vec<mpsc::Sender<IndexerMessage>>,
}

impl IndexerStub {
    pub fn spawn(db: Arc<Database>, opt: IndexerOpt) -> (IndexerStub, Vec<JoinHandle<()>>) {
        let random_state = RandomState::new();
        let indexing = Arc::new(RwLock::new(HashMap::new()));
        let mut txs = Vec::with_capacity(opt.indexers);
        let mut join_handles = Vec::with_capacity(opt.indexers);
        for idx in 0..opt.indexers {
            let (tx, rx) = mpsc::channel(500);
            txs.push(tx);
            join_handles.push(tokio::spawn(
                IndexerActor {
                    idx,
                    rx,
                    indexing: Arc::clone(&indexing),
                    db: Arc::clone(&db),
                    lila: Lila::new(opt.clone()),
                }
                .run(),
            ));
        }
        (
            IndexerStub {
                db,
                random_state,
                indexing,
                txs,
            },
            join_handles,
        )
    }

    pub async fn index_player(&self, player: &UserId) -> Option<watch::Receiver<()>> {
        // Optimization: First try subscribing to an existing indexing run,
        // without acquiring a write lock.
        {
            let guard = self.indexing.read().await;
            if let Some(sender) = guard.get(player) {
                return Some(sender.subscribe());
            }
        }

        // Check player indexing status.
        let mut status = self
            .db
            .queryable()
            .get_player_status(player)
            .expect("get player status")
            .unwrap_or_default();

        let since_created_at = match status
            .maybe_revisit_ongoing()
            .or_else(|| status.maybe_index())
        {
            Some(since) => since,
            None => return None, // Do not reindex so soon!
        };

        // Queue indexing request.
        let responsible_indexer = {
            let mut hasher = self.random_state.build_hasher();
            player.hash(&mut hasher);
            hasher.finish() as usize % self.txs.len()
        };

        let mut guard = self.indexing.write().await;
        let entry = match guard.entry(player.to_owned()) {
            Entry::Occupied(entry) => return Some(entry.get().subscribe()),
            Entry::Vacant(entry) => entry,
        };

        match self.txs[responsible_indexer].try_send(IndexerMessage::IndexPlayer {
            player: player.to_owned(),
            status,
            since_created_at,
        }) {
            Ok(_) => {
                let (sender, receiver) = watch::channel(());
                entry.insert(sender);
                Some(receiver)
            }
            Err(TrySendError::Full(_)) => {
                log::error!(
                    "indexer {}: not queuing {} because indexer queue is full",
                    responsible_indexer,
                    player.as_str()
                );
                None
            }
            Err(TrySendError::Closed(_)) => panic!("indexer {} died", responsible_indexer),
        }
    }
}

struct IndexerActor {
    idx: usize,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    rx: mpsc::Receiver<IndexerMessage>,
    db: Arc<Database>,
    lila: Lila,
}

impl IndexerActor {
    async fn run(mut self) {
        while let Some(msg) = self.rx.recv().await {
            match msg {
                IndexerMessage::IndexPlayer {
                    player,
                    status,
                    since_created_at,
                } => {
                    self.index_player(&player, status, since_created_at).await;

                    let mut guard = self.indexing.write().await;
                    guard.remove(&player);
                }
            }
        }
    }

    async fn index_player(
        &self,
        player: &UserId,
        mut status: PersonalStatus,
        since_created_at: u64,
    ) {
        log::info!(
            "indexer {} starting {} (created_at >= {})",
            self.idx,
            player.as_str(),
            since_created_at
        );
        let mut games = match self.lila.user_games(player, since_created_at).await {
            Ok(games) => games,
            Err(err) if err.status() == Some(StatusCode::NOT_FOUND) => {
                log::warn!("indexer did not find player {}", player.as_str());
                return;
            }
            Err(err) => {
                log::error!("indexer {}: request failed: {}", self.idx, err);
                return;
            }
        };

        let hash = ByColor::new_with(|color| PersonalKeyBuilder::with_user_pov(&player, color));
        let mut num_games = 0;
        while let Some(game) = games.next().await {
            let game = match game {
                Ok(game) => game,
                Err(err) => {
                    log::error!("{}", err);
                    continue;
                }
            };

            self.index_game(player, &hash, game, &mut status);

            num_games += 1;
            if num_games % 1024 == 0 {
                log::info!(
                    "indexer {}: indexed {} games for {}",
                    self.idx,
                    num_games,
                    player.as_str()
                );
            }
        }

        status.indexed_at = SystemTime::now();
        self.db
            .queryable()
            .put_player_status(player, status)
            .expect("put player status");
        log::info!(
            "indexer {}: finished indexing {} games for {}",
            self.idx,
            num_games,
            player.as_str()
        );
    }

    fn index_game(
        &self,
        player: &UserId,
        hash: &ByColor<PersonalKeyBuilder>,
        game: Game,
        status: &mut PersonalStatus,
    ) {
        status.latest_created_at = game.created_at;

        if game.status.is_ongoing() {
            if status.revisit_ongoing_created_at.is_none() {
                log::debug!("will revisit ongoing game {} eventually", game.id);
                status.revisit_ongoing_created_at = Some(game.created_at);
            }
            return;
        }

        if game.status.is_unindexable() {
            log::debug!("not indexing {} with status {:?}", game.id, game.status);
            return;
        }

        if game.players.any(|p| p.user.is_none()) {
            return;
        }

        let color = match game
            .players
            .find(|p| p.user.as_ref().map_or(false, |user| user.name == *player))
        {
            Some(color) => color,
            None => {
                log::error!("{} did not play in {}", player.as_str(), game.id);
                return;
            }
        };

        // Skip game if already indexed from this side. This cannot race with
        // writes, because all writes for the same player are sequenced by
        // this actor. So making a transaction is not required.
        let queryable = self.db.queryable();
        if queryable
            .get_game_info(game.id)
            .expect("get game info")
            .map_or(false, |info| info.indexed.into_color(color))
        {
            log::debug!("{}/{} already indexed", game.id, color);
            return;
        }

        // Prepare basic information and setup initial position.
        let month = Month::from_time_saturating(game.last_move_at);
        let outcome = Outcome::from_winner(game.winner);
        let variant = game.variant.into();
        let pos = match game.initial_fen {
            Some(fen) => VariantPosition::from_setup(variant, &fen, CastlingMode::Chess960),
            None => Ok(VariantPosition::new(variant)),
        };

        let mut pos: Zobrist<_, u128> = match pos {
            Ok(pos) => Zobrist::new(pos),
            Err(err) => {
                log::warn!("not indexing {}: {}", game.id, err);
                return;
            }
        };

        // Build an intermediate table to remove loops (due to repetitions).
        let mut table: FxHashMap<u128, Uci> =
            FxHashMap::with_capacity_and_hasher(game.moves.len(), Default::default());

        for (ply, san) in game.moves.into_iter().enumerate() {
            let m = match san.to_move(&pos) {
                Ok(m) => m,
                Err(err) => {
                    log::warn!("cutting off {} at ply {}: {}: {}", game.id, ply, err, san);
                    break;
                }
            };

            let uci = m.to_uci(CastlingMode::Chess960);
            table.insert(pos.zobrist_hash(), uci);

            pos.play_unchecked(&m);
        }

        // Write to database. All writes regarding this game are batched and
        // atomically committed, so the database will always be in a consistent
        // state.
        let mut batch = queryable.batch();

        batch.merge_game_info(
            game.id,
            GameInfo {
                winner: outcome.winner(),
                speed: game.speed,
                rated: game.rated,
                month,
                players: game.players.map(|p| GameInfoPlayer {
                    name: p.user.map(|p| p.name.to_string()),
                    rating: p.rating,
                }),
                indexed: ByColor::new_with(|c| color == c),
            },
        );

        for (zobrist, uci) in table {
            batch.merge_personal(
                hash.by_color(color)
                    .with_zobrist(variant, zobrist)
                    .with_month(month),
                PersonalEntry::new_single(
                    uci.clone(),
                    game.speed,
                    Mode::from_rated(game.rated),
                    game.id,
                    outcome,
                ),
            );
        }

        batch.write().expect("atomically commit game and moves");
    }
}

enum IndexerMessage {
    IndexPlayer {
        player: UserId,
        status: PersonalStatus,
        since_created_at: u64,
    },
}
<|endoftext|>


<commit_before>
use std::fmt;
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug, Default)]
pub struct Interpreter {
    // File data
    header: RaptorHeader,
    bytecode: Vec<u8>,

    // Runtime stuff
    stack: Vec<i32>,
    memory: Vec<i32>,
}

// TODO: Use more slices
impl Interpreter {
    pub fn new(mut data: Vec<u8>) -> Interpreter {
        let i = Interpreter {
            header: read_header(&data),
            bytecode: data.drain(HEADER_SIZE..).collect(),
            stack: Vec::new(),
            memory: Vec::new(),
        };

        i
    }

    pub fn run(&mut self) {
        debug!("Running...");
    }
}

// TODO: Move all header stuff to another module

const HEADER_SIZE: usize = 8;
const HEADER_OFFSET: usize = 0;
const MAGIC_VALUE: u32 = 0x5AB70500;

#[derive(Default)]
pub struct RaptorHeader {
    magic: u32,     // Magic number + padding ( 0x5AB70500 )
    var_count: u32,     // Number of variables
}

impl RaptorHeader {
    fn verify(&self) -> bool {
        self.magic == MAGIC_VALUE
    }
}

impl fmt::Debug for RaptorHeader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "RaptorHeader {{
            magic: 0x{:04X}
            var_count: {}
        }}",
            self.magic,
            self.var_count
        )
    }
}

pub fn read_header(data: &Vec<u8>) -> RaptorHeader {
    if data.len() < HEADER_SIZE  {
        panic!("Invalid header size");
    }

    let header = read_header_impl(&data);
    if !header.verify() {
        panic!("Invalid header magic");
    }

    debug!("Header verified");
    header
}

fn read_header_impl(data: &Vec<u8>) -> RaptorHeader {
    use std::slice;
    use std::io::Read;

    let mut buffer: [u8; HEADER_SIZE] = [0u8; HEADER_SIZE];

    for i in 0..HEADER_SIZE {
        buffer[i] = data[i + HEADER_OFFSET];
    }

    let mut buffer_slice: &[u8] = &buffer;

    let mut header: RaptorHeader = Default::default();

    header.magic = buffer_slice.read_u32::<BigEndian>().unwrap();
    header.var_count = buffer_slice.read_u32::<BigEndian>().unwrap();

    print!("{:04X}",header.magic);
    debug!("Read header: {:#?}", header);
    header
}

// TODO: Add tests
<commit_msg>Add a test for header size<commit_after>
use std::fmt;
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug, Default)]
pub struct Interpreter {
    // File data
    header: RaptorHeader,
    bytecode: Vec<u8>,

    // Runtime stuff
    stack: Vec<i32>,
    memory: Vec<i32>,
}

// TODO: Use more slices
impl Interpreter {
    pub fn new(mut data: Vec<u8>) -> Interpreter {
        let i = Interpreter {
            header: read_header(&data),
            bytecode: data.drain(HEADER_SIZE..).collect(),
            stack: Vec::new(),
            memory: Vec::new(),
        };

        i
    }

    pub fn run(&mut self) {
        debug!("Running...");
    }
}

// TODO: Move all header stuff to another module

pub const HEADER_SIZE: usize = 8;
const MAGIC_VALUE: u32 = 0x5AB70500;

#[derive(Default)]
pub struct RaptorHeader {
    magic: u32,     // Magic number + padding ( 0x5AB70500 )
    var_count: u32,     // Number of variables
}

impl RaptorHeader {
    fn verify(&self) -> bool {
        self.magic == MAGIC_VALUE
    }
}

impl fmt::Debug for RaptorHeader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "RaptorHeader {{
            magic: 0x{:04X}
            var_count: {}
        }}",
            self.magic,
            self.var_count
        )
    }
}

pub fn read_header(data: &Vec<u8>) -> RaptorHeader {
    if data.len() < HEADER_SIZE  {
        panic!("Invalid header size");
    }

    let header = read_header_impl(&data);
    if !header.verify() {
        panic!("Invalid header magic");
    }

    debug!("Header verified");
    header
}

fn read_header_impl(data: &Vec<u8>) -> RaptorHeader {
    let mut buffer: [u8; HEADER_SIZE] = [0u8; HEADER_SIZE];

    for i in 0..HEADER_SIZE {
        buffer[i] = data[i];
    }

    let mut buffer_slice: &[u8] = &buffer;

    let mut header: RaptorHeader = Default::default();

    header.magic = buffer_slice.read_u32::<BigEndian>().unwrap();
    header.var_count = buffer_slice.read_u32::<BigEndian>().unwrap();

    print!("{:04X}",header.magic);
    debug!("Read header: {:#?}", header);
    header
}

#[cfg(test)]
mod header_tests {
    use super::*;

    #[test]
    fn header_size() {
        use std::mem;

        assert_eq!(HEADER_SIZE, mem::size_of::<RaptorHeader>());
    }
}<|endoftext|>