<commit_before>//! This pass adds validation calls (AcquireValid, ReleaseValid) where appropriate.
//! It has to be run really early, before transformations like inlining, because
//! introducing these calls *adds* UB -- so, conceptually, this pass is actually part
//! of MIR building, and only after this pass we think of the program has having the
//! normal MIR semantics.

use crate::transform::MirPass;
use rustc_middle::mir::*;
use rustc_middle::ty::{self, Ty, TyCtxt};

pub struct AddRetag;

/// Determines whether this place is "stable": Whether, if we evaluate it again
/// after the assignment, we can be sure to obtain the same place value.
/// (Concurrent accesses by other threads are no problem as these are anyway non-atomic
/// copies.  Data races are UB.)
fn is_stable(place: PlaceRef<'_>) -> bool {
    place.projection.iter().all(|elem| {
        match elem {
            // Which place this evaluates to can change with any memory write,
            // so cannot assume this to be stable.
            ProjectionElem::Deref => false,
            // Array indices are interesting, but MIR building generates a *fresh*
            // temporary for every array access, so the index cannot be changed as
            // a side-effect.
            ProjectionElem::Index { .. } |
            // The rest is completely boring, they just offset by a constant.
            ProjectionElem::Field { .. } |
            ProjectionElem::ConstantIndex { .. } |
            ProjectionElem::Subslice { .. } |
            ProjectionElem::Downcast { .. } => true,
        }
    })
}

/// Determine whether this type may be a reference (or box), and thus needs retagging.
fn may_be_reference(ty: Ty<'tcx>) -> bool {
    match ty.kind() {
        // Primitive types that are not references
        ty::Bool
        | ty::Char
        | ty::Float(_)
        | ty::Int(_)
        | ty::Uint(_)
        | ty::RawPtr(..)
        | ty::FnPtr(..)
        | ty::Str
        | ty::FnDef(..)
        | ty::Never => false,
        // References
        ty::Ref(..) => true,
        ty::Adt(..) if ty.is_box() => true,
        // Compound types are not references
        ty::Array(..) | ty::Slice(..) | ty::Tuple(..) | ty::Adt(..) => false,
        // Conservative fallback
        _ => true,
    }
}

impl<'tcx> MirPass<'tcx> for AddRetag {
    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
        if !tcx.sess.opts.debugging_opts.mir_emit_retag {
            return;
        }

        // We need an `AllCallEdges` pass before we can do any work.
        super::add_call_guards::AllCallEdges.run_pass(tcx, body);

        let (span, arg_count) = (body.span, body.arg_count);
        let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut();
        let needs_retag = |place: &Place<'tcx>| {
            // FIXME: Instead of giving up for unstable places, we should introduce
            // a temporary and retag on that.
            is_stable(place.as_ref()) && may_be_reference(place.ty(&*local_decls, tcx).ty)
        };

        // PART 1
        // Retag arguments at the beginning of the start block.
        {
            // FIXME: Consider using just the span covering the function
            // argument declaration.
            let source_info = SourceInfo::outermost(span);
            // Gather all arguments, skip return value.
            let places = local_decls
                .iter_enumerated()
                .skip(1)
                .take(arg_count)
                .map(|(local, _)| Place::from(local))
                .filter(needs_retag);
            // Emit their retags.
            basic_blocks[START_BLOCK].statements.splice(
                0..0,
                places.map(|place| Statement {
                    source_info,
                    kind: StatementKind::Retag(RetagKind::FnEntry, box (place)),
                }),
            );
        }

        // PART 2
        // Retag return values of functions.  Also escape-to-raw the argument of `drop`.
        // We collect the return destinations because we cannot mutate while iterating.
        let returns = basic_blocks
            .iter_mut()
            .filter_map(|block_data| {
                match block_data.terminator().kind {
                    TerminatorKind::Call { destination: Some(ref destination), .. }
                        if needs_retag(&destination.0) =>
                    {
                        // Remember the return destination for later
                        Some((block_data.terminator().source_info, destination.0, destination.1))
                    }

                    // `Drop` is also a call, but it doesn't return anything so we are good.
                    TerminatorKind::Drop { .. } | TerminatorKind::DropAndReplace { .. } => None,
                    // Not a block ending in a Call -> ignore.
                    _ => None,
                }
            })
            .collect::<Vec<_>>();
        // Now we go over the returns we collected to retag the return values.
        for (source_info, dest_place, dest_block) in returns {
            basic_blocks[dest_block].statements.insert(
                0,
                Statement {
                    source_info,
                    kind: StatementKind::Retag(RetagKind::Default, box (dest_place)),
                },
            );
        }

        // PART 3
        // Add retag after assignment.
        for block_data in basic_blocks {
            // We want to insert statements as we iterate.  To this end, we
            // iterate backwards using indices.
            for i in (0..block_data.statements.len()).rev() {
                let (retag_kind, place) = match block_data.statements[i].kind {
                    // Retag-as-raw after escaping to a raw pointer.
                    StatementKind::Assign(box (place, Rvalue::AddressOf(..))) => {
                        (RetagKind::Raw, place)
                    }
                    // Assignments of reference or ptr type are the ones where we may have
                    // to update tags.  This includes `x = &[mut] ...` and hence
                    // we also retag after taking a reference!
                    StatementKind::Assign(box (ref place, ref rvalue)) if needs_retag(place) => {
                        let kind = match rvalue {
                            Rvalue::Ref(_, borrow_kind, _)
                                if borrow_kind.allows_two_phase_borrow() =>
                            {
                                RetagKind::TwoPhase
                            }
                            _ => RetagKind::Default,
                        };
                        (kind, *place)
                    }
                    // Do nothing for the rest
                    _ => continue,
                };
                // Insert a retag after the statement.
                let source_info = block_data.statements[i].source_info;
                block_data.statements.insert(
                    i + 1,
                    Statement { source_info, kind: StatementKind::Retag(retag_kind, box (place)) },
                );
            }
        }
    }
}
<commit_msg>Retagging: do not retag 'raw reborrows'<commit_after>//! This pass adds validation calls (AcquireValid, ReleaseValid) where appropriate.
//! It has to be run really early, before transformations like inlining, because
//! introducing these calls *adds* UB -- so, conceptually, this pass is actually part
//! of MIR building, and only after this pass we think of the program has having the
//! normal MIR semantics.

use crate::transform::MirPass;
use rustc_middle::mir::*;
use rustc_middle::ty::{self, Ty, TyCtxt};

pub struct AddRetag;

/// Determines whether this place is "stable": Whether, if we evaluate it again
/// after the assignment, we can be sure to obtain the same place value.
/// (Concurrent accesses by other threads are no problem as these are anyway non-atomic
/// copies.  Data races are UB.)
fn is_stable(place: PlaceRef<'_>) -> bool {
    place.projection.iter().all(|elem| {
        match elem {
            // Which place this evaluates to can change with any memory write,
            // so cannot assume this to be stable.
            ProjectionElem::Deref => false,
            // Array indices are interesting, but MIR building generates a *fresh*
            // temporary for every array access, so the index cannot be changed as
            // a side-effect.
            ProjectionElem::Index { .. } |
            // The rest is completely boring, they just offset by a constant.
            ProjectionElem::Field { .. } |
            ProjectionElem::ConstantIndex { .. } |
            ProjectionElem::Subslice { .. } |
            ProjectionElem::Downcast { .. } => true,
        }
    })
}

/// Determine whether this type may be a reference (or box), and thus needs retagging.
fn may_be_reference(ty: Ty<'tcx>) -> bool {
    match ty.kind() {
        // Primitive types that are not references
        ty::Bool
        | ty::Char
        | ty::Float(_)
        | ty::Int(_)
        | ty::Uint(_)
        | ty::RawPtr(..)
        | ty::FnPtr(..)
        | ty::Str
        | ty::FnDef(..)
        | ty::Never => false,
        // References
        ty::Ref(..) => true,
        ty::Adt(..) if ty.is_box() => true,
        // Compound types are not references
        ty::Array(..) | ty::Slice(..) | ty::Tuple(..) | ty::Adt(..) => false,
        // Conservative fallback
        _ => true,
    }
}

impl<'tcx> MirPass<'tcx> for AddRetag {
    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
        if !tcx.sess.opts.debugging_opts.mir_emit_retag {
            return;
        }

        // We need an `AllCallEdges` pass before we can do any work.
        super::add_call_guards::AllCallEdges.run_pass(tcx, body);

        let (span, arg_count) = (body.span, body.arg_count);
        let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut();
        let needs_retag = |place: &Place<'tcx>| {
            // FIXME: Instead of giving up for unstable places, we should introduce
            // a temporary and retag on that.
            is_stable(place.as_ref()) && may_be_reference(place.ty(&*local_decls, tcx).ty)
        };
        let place_base_raw = |place: &Place<'tcx>| {
            // If this is a `Deref`, get the type of what we are deref'ing.
            let deref_base =
                place.projection.iter().rposition(|p| matches!(p, ProjectionElem::Deref));
            if let Some(deref_base) = deref_base {
                let base_proj = &place.projection[..deref_base];
                let ty = Place::ty_from(place.local, base_proj, &*local_decls, tcx).ty;
                ty.is_unsafe_ptr()
            } else {
                // Not a deref, and thus not raw.
                false
            }
        };

        // PART 1
        // Retag arguments at the beginning of the start block.
        {
            // FIXME: Consider using just the span covering the function
            // argument declaration.
            let source_info = SourceInfo::outermost(span);
            // Gather all arguments, skip return value.
            let places = local_decls
                .iter_enumerated()
                .skip(1)
                .take(arg_count)
                .map(|(local, _)| Place::from(local))
                .filter(needs_retag);
            // Emit their retags.
            basic_blocks[START_BLOCK].statements.splice(
                0..0,
                places.map(|place| Statement {
                    source_info,
                    kind: StatementKind::Retag(RetagKind::FnEntry, box (place)),
                }),
            );
        }

        // PART 2
        // Retag return values of functions.  Also escape-to-raw the argument of `drop`.
        // We collect the return destinations because we cannot mutate while iterating.
        let returns = basic_blocks
            .iter_mut()
            .filter_map(|block_data| {
                match block_data.terminator().kind {
                    TerminatorKind::Call { destination: Some(ref destination), .. }
                        if needs_retag(&destination.0) =>
                    {
                        // Remember the return destination for later
                        Some((block_data.terminator().source_info, destination.0, destination.1))
                    }

                    // `Drop` is also a call, but it doesn't return anything so we are good.
                    TerminatorKind::Drop { .. } | TerminatorKind::DropAndReplace { .. } => None,
                    // Not a block ending in a Call -> ignore.
                    _ => None,
                }
            })
            .collect::<Vec<_>>();
        // Now we go over the returns we collected to retag the return values.
        for (source_info, dest_place, dest_block) in returns {
            basic_blocks[dest_block].statements.insert(
                0,
                Statement {
                    source_info,
                    kind: StatementKind::Retag(RetagKind::Default, box (dest_place)),
                },
            );
        }

        // PART 3
        // Add retag after assignment.
        for block_data in basic_blocks {
            // We want to insert statements as we iterate.  To this end, we
            // iterate backwards using indices.
            for i in (0..block_data.statements.len()).rev() {
                let (retag_kind, place) = match block_data.statements[i].kind {
                    // Retag-as-raw after escaping to a raw pointer, if the referent
                    // is not already a raw pointer.
                    StatementKind::Assign(box (lplace, Rvalue::AddressOf(_, ref rplace)))
                        if !place_base_raw(rplace) =>
                    {
                        (RetagKind::Raw, lplace)
                    }
                    // Retag after assignments of reference type.
                    StatementKind::Assign(box (ref place, ref rvalue)) if needs_retag(place) => {
                        let kind = match rvalue {
                            Rvalue::Ref(_, borrow_kind, _)
                                if borrow_kind.allows_two_phase_borrow() =>
                            {
                                RetagKind::TwoPhase
                            }
                            _ => RetagKind::Default,
                        };
                        (kind, *place)
                    }
                    // Do nothing for the rest
                    _ => continue,
                };
                // Insert a retag after the statement.
                let source_info = block_data.statements[i].source_info;
                block_data.statements.insert(
                    i + 1,
                    Statement { source_info, kind: StatementKind::Retag(retag_kind, box (place)) },
                );
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::rc::Rc;
use std::str::FromStr;
use std::time::Duration;

use futures;
use futures::{Future, Stream, Poll};
use futures::future::{result};
use hyper;
use hyper::{Body, Method, Uri};
use hyper::client::Client;
use hyper::header::ContentType;
use hyper_tls::HttpsConnector;
use serde_json;
use tokio_core::reactor::{Handle, Timeout};

use telegram_bot_raw::{Request, Response};

use errors::{Error, Result, ErrorKind};

use stream::UpdatesStream;

const TELEGRAM_URL: &'static str = "https://api.telegram.org/";

#[must_use = "futures do nothing unless polled"]
pub struct TelegramFuture<T> {
    inner: Box<Future<Item=T, Error=Error>>
}

impl<T> Future for TelegramFuture<T> {
    type Item = T;
    type Error = Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        self.inner.poll()
    }
}

#[derive(Clone)]
pub struct Api {
    inner: Rc<ApiInner>,
}

#[derive(Clone)]
struct ApiInner {
    token: String,
    client: Client<HttpsConnector>,
    handle: Handle,
}

impl Api {
    pub fn from_token(handle: &Handle, token: &str) -> Result<Self> {
        let connector = HttpsConnector::new(1, handle);
        let config = Client::configure().connector(connector);

        Ok(Api {
            inner: Rc::new(ApiInner {
                token: token.to_string(),
                client: config.build(handle),
                handle: handle.clone(),
            }),
        })
    }

    pub fn stream(&self) -> UpdatesStream {
        UpdatesStream::new(self)
    }

    pub fn spawn<Req>(&self, request: &Req)
        where Req: Request + 'static, <Req as Request>::Response: ::std::marker::Send + 'static {

        self.inner.handle.spawn(self.send(request).then(|_| Ok(())))
    }

    pub fn send_timeout<Req>(
        &self, request: &Req, duration: Duration) -> TelegramFuture<Option<Req::Response>>
        where Req: Request + 'static, <Req as Request>::Response: ::std::marker::Send + 'static {

        let timeout_future = result(Timeout::new(duration, &self.inner.handle))
            .flatten().map_err(From::from).map(|()| None);
        let send_future = self.send(request).map(|resp| Some(resp));

        let future = timeout_future.select(send_future)
            .map(|(item, _next)| item)
            .map_err(|(item, _next)| item);

        TelegramFuture {
            inner: Box::new(future)
        }
    }

    pub fn send<Req>(&self, request: &Req) -> TelegramFuture<Req::Response>
        where Req: Request + 'static, <Req as Request>::Response: ::std::marker::Send + 'static {

        let name = Req::name();
        let encoded = serde_json::to_vec(&request);

        let url = result(url(&self.inner.token, name));
        let body = futures::lazy(move || {
            encoded.map(Body::from)
        }).map_err(From::from);

        let api = self.clone();
        let response = url.join(body).and_then(move |(url, body)| {
            let mut http_request = hyper::client::Request::new(Method::Post, url);
            http_request.set_body(body);
            http_request.headers_mut().set(ContentType::json());

            api.inner.client.request(http_request).map_err(From::from)
        });

        let bytes = response.and_then(|response| {
            response.body().map_err(From::from)
                .fold(vec![], |mut result, chunk| -> Result<Vec<u8>> {
                    result.extend_from_slice(&chunk);
                    Ok(result)
            })
        });

        let future = bytes.and_then(|bytes| {
            result(serde_json::from_slice(&bytes).map_err(From::from).and_then(|value| {
                match value {
                    Response::Success {result} => Ok(Req::map(result)),
                    Response::Error { description, parameters } => {
                        Err(ErrorKind::TelegramError {
                            description: description,
                            parameters: parameters
                        }.into())
                    },
                }
            }))
        });

        TelegramFuture {
            inner: Box::new(future)
        }
    }
}

fn url(token: &str, method: &str) -> Result<Uri> {
    Uri::from_str(&format!("{}bot{}/{}", TELEGRAM_URL, token, method)).map_err(From::from)
}
<commit_msg>refactor: take Request by value in Api::send<commit_after>use std::rc::Rc;
use std::str::FromStr;
use std::time::Duration;

use futures;
use futures::{Future, Stream, Poll};
use futures::future::{result};
use hyper;
use hyper::{Body, Method, Uri};
use hyper::client::Client;
use hyper::header::ContentType;
use hyper_tls::HttpsConnector;
use serde_json;
use tokio_core::reactor::{Handle, Timeout};

use telegram_bot_raw::{Request, Response};

use errors::{Error, Result, ErrorKind};

use stream::UpdatesStream;

const TELEGRAM_URL: &'static str = "https://api.telegram.org/";

#[must_use = "futures do nothing unless polled"]
pub struct TelegramFuture<T> {
    inner: Box<Future<Item=T, Error=Error>>
}

impl<T> Future for TelegramFuture<T> {
    type Item = T;
    type Error = Error;

    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        self.inner.poll()
    }
}

#[derive(Clone)]
pub struct Api {
    inner: Rc<ApiInner>,
}

#[derive(Clone)]
struct ApiInner {
    token: String,
    client: Client<HttpsConnector>,
    handle: Handle,
}

impl Api {
    pub fn from_token(handle: &Handle, token: &str) -> Result<Self> {
        let connector = HttpsConnector::new(1, handle);
        let config = Client::configure().connector(connector);

        Ok(Api {
            inner: Rc::new(ApiInner {
                token: token.to_string(),
                client: config.build(handle),
                handle: handle.clone(),
            }),
        })
    }

    pub fn stream(&self) -> UpdatesStream {
        UpdatesStream::new(self)
    }

    pub fn spawn<Req>(&self, request: Req)
        where Req: Request, <Req as Request>::Response: ::std::marker::Send + 'static {

        self.inner.handle.spawn(self.send(request).then(|_| Ok(())))
    }

    pub fn send_timeout<Req>(
        &self, request: Req, duration: Duration) -> TelegramFuture<Option<Req::Response>>
        where Req: Request, <Req as Request>::Response: ::std::marker::Send + 'static {

        let timeout_future = result(Timeout::new(duration, &self.inner.handle))
            .flatten().map_err(From::from).map(|()| None);
        let send_future = self.send(request).map(|resp| Some(resp));

        let future = timeout_future.select(send_future)
            .map(|(item, _next)| item)
            .map_err(|(item, _next)| item);

        TelegramFuture {
            inner: Box::new(future)
        }
    }

    pub fn send<Req>(&self, request: Req) -> TelegramFuture<Req::Response>
        where Req: Request, <Req as Request>::Response: ::std::marker::Send + 'static {

        let name = Req::name();
        let encoded = serde_json::to_vec(&request);

        let url = result(url(&self.inner.token, name));
        let body = futures::lazy(move || {
            encoded.map(Body::from)
        }).map_err(From::from);

        let api = self.clone();
        let response = url.join(body).and_then(move |(url, body)| {
            let mut http_request = hyper::client::Request::new(Method::Post, url);
            http_request.set_body(body);
            http_request.headers_mut().set(ContentType::json());

            api.inner.client.request(http_request).map_err(From::from)
        });

        let bytes = response.and_then(|response| {
            response.body().map_err(From::from)
                .fold(vec![], |mut result, chunk| -> Result<Vec<u8>> {
                    result.extend_from_slice(&chunk);
                    Ok(result)
            })
        });

        let future = bytes.and_then(|bytes| {
            result(serde_json::from_slice(&bytes).map_err(From::from).and_then(|value| {
                match value {
                    Response::Success {result} => Ok(Req::map(result)),
                    Response::Error { description, parameters } => {
                        Err(ErrorKind::TelegramError {
                            description: description,
                            parameters: parameters
                        }.into())
                    },
                }
            }))
        });

        TelegramFuture {
            inner: Box::new(future)
        }
    }
}

fn url(token: &str, method: &str) -> Result<Uri> {
    Uri::from_str(&format!("{}bot{}/{}", TELEGRAM_URL, token, method)).map_err(From::from)
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

<%namespace name="helpers" file="/helpers.mako.rs" />

<%helpers:shorthand name="overflow" sub_properties="overflow-x overflow-y"
                    spec="https://drafts.csswg.org/css-overflow/#propdef-overflow">
    use properties::longhands::overflow_x::parse as parse_overflow;
    % if product == "gecko":
        use properties::longhands::overflow_x::SpecifiedValue;
    % endif

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        % if product == "gecko":
            let moz_kw_found = input.try(|i| {
                try_match_ident_ignore_ascii_case! { i.expect_ident()?,
                    "-moz-scrollbars-horizontal" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::scroll,
                            overflow_y: SpecifiedValue::hidden,
                        })
                    }
                    "-moz-scrollbars-vertical" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::hidden,
                            overflow_y: SpecifiedValue::scroll,
                        })
                    }
                    "-moz-scrollbars-none" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::hidden,
                            overflow_y: SpecifiedValue::hidden,
                        })
                    }
                }
            });
            if moz_kw_found.is_ok() {
                return moz_kw_found
            }
        % endif
        let overflow = parse_overflow(context, input)?;
        Ok(expanded! {
            overflow_x: overflow,
            overflow_y: overflow,
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            if self.overflow_x == self.overflow_y {
                self.overflow_x.to_css(dest)
            } else {
                Ok(())
            }
        }
    }
</%helpers:shorthand>

macro_rules! try_parse_one {
    ($input: expr, $var: ident, $prop_module: ident) => {
        if $var.is_none() {
            if let Ok(value) = $input.try($prop_module::SingleSpecifiedValue::parse) {
                $var = Some(value);
                continue;
            }
        }
    };
    ($context: expr, $input: expr, $var: ident, $prop_module: ident) => {
        if $var.is_none() {
            if let Ok(value) = $input.try(|i| {
                $prop_module::SingleSpecifiedValue::parse($context, i)
            }) {
                $var = Some(value);
                continue;
            }
        }
    };
}

<%helpers:shorthand name="transition" extra_prefixes="moz webkit"
                    sub_properties="transition-property transition-duration
                                    transition-timing-function
                                    transition-delay"
                    spec="https://drafts.csswg.org/css-transitions/#propdef-transition">
    use parser::Parse;
    % for prop in "delay duration property timing_function".split():
    use properties::longhands::transition_${prop};
    % endfor

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        struct SingleTransition {
            % for prop in "property duration timing_function delay".split():
            transition_${prop}: transition_${prop}::SingleSpecifiedValue,
            % endfor
        }

        fn parse_one_transition<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                                        -> Result<SingleTransition,ParseError<'i>> {
            % for prop in "property duration timing_function delay".split():
            let mut ${prop} = None;
            % endfor

            let mut parsed = 0;
            loop {
                parsed += 1;

                try_parse_one!(context, input, duration, transition_duration);
                try_parse_one!(context, input, timing_function, transition_timing_function);
                try_parse_one!(context, input, delay, transition_delay);
                // Must check 'transition-property' after 'transition-timing-function' since
                // 'transition-property' accepts any keyword.
                try_parse_one!(input, property, transition_property);

                parsed -= 1;
                break
            }

            if parsed != 0 {
                Ok(SingleTransition {
                    % for prop in "property duration timing_function delay".split():
                    transition_${prop}: ${prop}.unwrap_or_else(transition_${prop}::single_value
                                                                                ::get_initial_specified_value),
                    % endfor
                })
            } else {
                Err(StyleParseError::UnspecifiedError.into())
            }
        }

        % for prop in "property duration timing_function delay".split():
        let mut ${prop}s = Vec::new();
        % endfor

        if input.try(|input| input.expect_ident_matching("none")).is_err() {
            let results = input.parse_comma_separated(|i| parse_one_transition(context, i))?;
            for result in results {
                % for prop in "property duration timing_function delay".split():
                ${prop}s.push(result.transition_${prop});
                % endfor
            }
        } else {
            // `transition: none` is a valid syntax, and we keep transition_property empty because |none| is not
            // a valid TransitionProperty.
            // durations, delays, and timing_functions are not allowed as empty, so before we convert them into
            // longhand properties, we need to put initial values for none transition.
            % for prop in "duration timing_function delay".split():
            ${prop}s.push(transition_${prop}::single_value::get_initial_specified_value());
            % endfor
        }

        Ok(expanded! {
            % for prop in "property duration timing_function delay".split():
            transition_${prop}: transition_${prop}::SpecifiedValue(${prop}s),
            % endfor
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            let len = self.transition_property.0.len();
            // There should be at least one declared value
            if len == 0 {
                return Ok(());
            }

            // If any value list length is differs then we don't do a shorthand serialization
            // either.
            % for name in "property duration delay timing_function".split():
                if len != self.transition_${name}.0.len() {
                    return Ok(());
                }
            % endfor

            for i in 0..len {
                if i != 0 {
                    dest.write_str(", ")?;
                }
                self.transition_property.0[i].to_css(dest)?;
                % for name in "duration timing_function delay".split():
                    dest.write_str(" ")?;
                    self.transition_${name}.0[i].to_css(dest)?;
                % endfor
            }
            Ok(())
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="animation" extra_prefixes="moz webkit"
                    sub_properties="animation-name animation-duration
                                    animation-timing-function animation-delay
                                    animation-iteration-count animation-direction
                                    animation-fill-mode animation-play-state"
                    allowed_in_keyframe_block="False"
                    spec="https://drafts.csswg.org/css-animations/#propdef-animation">
    <%
        props = "name duration timing_function delay iteration_count \
                 direction fill_mode play_state".split()
    %>
    use parser::Parse;
    % for prop in props:
    use properties::longhands::animation_${prop};
    % endfor

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        struct SingleAnimation {
            % for prop in props:
            animation_${prop}: animation_${prop}::SingleSpecifiedValue,
            % endfor
        }

        fn parse_one_animation<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                                       -> Result<SingleAnimation,ParseError<'i>> {
            % for prop in props:
            let mut ${prop} = None;
            % endfor

            let mut parsed = 0;
            // NB: Name must be the last one here so that keywords valid for other
            // longhands are not interpreted as names.
            //
            // Also, duration must be before delay, see
            // https://drafts.csswg.org/css-animations/#typedef-single-animation
            loop {
                parsed += 1;
                try_parse_one!(context, input, duration, animation_duration);
                try_parse_one!(context, input, timing_function, animation_timing_function);
                try_parse_one!(context, input, delay, animation_delay);
                try_parse_one!(context, input, iteration_count, animation_iteration_count);
                try_parse_one!(input, direction, animation_direction);
                try_parse_one!(input, fill_mode, animation_fill_mode);
                try_parse_one!(input, play_state, animation_play_state);
                try_parse_one!(context, input, name, animation_name);

                parsed -= 1;
                break
            }

            // If nothing is parsed, this is an invalid entry.
            if parsed == 0 {
                Err(StyleParseError::UnspecifiedError.into())
            } else {
                Ok(SingleAnimation {
                    % for prop in props:
                    animation_${prop}: ${prop}.unwrap_or_else(animation_${prop}::single_value
                                                              ::get_initial_specified_value),
                    % endfor
                })
            }
        }

        % for prop in props:
        let mut ${prop}s = vec![];
        % endfor

        let results = input.parse_comma_separated(|i| parse_one_animation(context, i))?;
        for result in results.into_iter() {
            % for prop in props:
            ${prop}s.push(result.animation_${prop});
            % endfor
        }

        Ok(expanded! {
            % for prop in props:
            animation_${prop}: animation_${prop}::SpecifiedValue(${prop}s),
            % endfor
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            let len = self.animation_name.0.len();
            // There should be at least one declared value
            if len == 0 {
                return Ok(());
            }

            // If any value list length is differs then we don't do a shorthand serialization
            // either.
            % for name in props[1:]:
                if len != self.animation_${name}.0.len() {
                    return Ok(())
                }
            % endfor

            for i in 0..len {
                if i != 0 {
                    dest.write_str(", ")?;
                }

                % for name in props[1:]:
                    self.animation_${name}.0[i].to_css(dest)?;
                    dest.write_str(" ")?;
                % endfor
                self.animation_name.0[i].to_css(dest)?;
            }
            Ok(())
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="scroll-snap-type" products="gecko"
                    sub_properties="scroll-snap-type-x scroll-snap-type-y"
                    spec="https://drafts.csswg.org/css-scroll-snap/#propdef-scroll-snap-type">
    use properties::longhands::scroll_snap_type_x;

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        let result = scroll_snap_type_x::parse(context, input)?;
        Ok(expanded! {
            scroll_snap_type_x: result,
            scroll_snap_type_y: result,
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        // Serializes into the single keyword value if both scroll-snap-type and scroll-snap-type-y are same.
        // Otherwise into an empty string. This is done to match Gecko's behaviour.
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            if self.scroll_snap_type_x == self.scroll_snap_type_y {
                self.scroll_snap_type_x.to_css(dest)
            } else {
                Ok(())
            }
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="-moz-transform" products="gecko"
                    sub_properties="transform"
                    flags="SHORTHAND_ALIAS_PROPERTY"
                    derive_serialize="True"
                    spec="Non-standard: https://developer.mozilla.org/en-US/docs/Web/CSS/transform">
    use properties::longhands::transform;

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        Ok(expanded! {
            transform: transform::parse_prefixed(context, input)?,
        })
    }
</%helpers:shorthand>
<commit_msg>Handle values as valid in single transition shorthand even if the transition-property is 'none'.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

<%namespace name="helpers" file="/helpers.mako.rs" />

<%helpers:shorthand name="overflow" sub_properties="overflow-x overflow-y"
                    spec="https://drafts.csswg.org/css-overflow/#propdef-overflow">
    use properties::longhands::overflow_x::parse as parse_overflow;
    % if product == "gecko":
        use properties::longhands::overflow_x::SpecifiedValue;
    % endif

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        % if product == "gecko":
            let moz_kw_found = input.try(|i| {
                try_match_ident_ignore_ascii_case! { i.expect_ident()?,
                    "-moz-scrollbars-horizontal" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::scroll,
                            overflow_y: SpecifiedValue::hidden,
                        })
                    }
                    "-moz-scrollbars-vertical" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::hidden,
                            overflow_y: SpecifiedValue::scroll,
                        })
                    }
                    "-moz-scrollbars-none" => {
                        Ok(expanded! {
                            overflow_x: SpecifiedValue::hidden,
                            overflow_y: SpecifiedValue::hidden,
                        })
                    }
                }
            });
            if moz_kw_found.is_ok() {
                return moz_kw_found
            }
        % endif
        let overflow = parse_overflow(context, input)?;
        Ok(expanded! {
            overflow_x: overflow,
            overflow_y: overflow,
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            if self.overflow_x == self.overflow_y {
                self.overflow_x.to_css(dest)
            } else {
                Ok(())
            }
        }
    }
</%helpers:shorthand>

macro_rules! try_parse_one {
    ($input: expr, $var: ident, $prop_module: ident) => {
        if $var.is_none() {
            if let Ok(value) = $input.try($prop_module::SingleSpecifiedValue::parse) {
                $var = Some(value);
                continue;
            }
        }
    };
    ($context: expr, $input: expr, $var: ident, $prop_module: ident) => {
        if $var.is_none() {
            if let Ok(value) = $input.try(|i| {
                $prop_module::SingleSpecifiedValue::parse($context, i)
            }) {
                $var = Some(value);
                continue;
            }
        }
    };
}

<%helpers:shorthand name="transition" extra_prefixes="moz webkit"
                    sub_properties="transition-property transition-duration
                                    transition-timing-function
                                    transition-delay"
                    spec="https://drafts.csswg.org/css-transitions/#propdef-transition">
    use parser::Parse;
    % for prop in "delay duration property timing_function".split():
    use properties::longhands::transition_${prop};
    % endfor

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        struct SingleTransition {
            % for prop in "duration timing_function delay".split():
            transition_${prop}: transition_${prop}::SingleSpecifiedValue,
            % endfor
            // Unlike other properties, transition-property uses an Option<> to
            // represent 'none' as `None`.
            transition_property: Option<transition_property::SingleSpecifiedValue>,
        }

        fn parse_one_transition<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                                        -> Result<SingleTransition,ParseError<'i>> {
            % for prop in "property duration timing_function delay".split():
            let mut ${prop} = None;
            % endfor

            let mut parsed = 0;
            loop {
                parsed += 1;

                try_parse_one!(context, input, duration, transition_duration);
                try_parse_one!(context, input, timing_function, transition_timing_function);
                try_parse_one!(context, input, delay, transition_delay);
                // Must check 'transition-property' after 'transition-timing-function' since
                // 'transition-property' accepts any keyword.
                if property.is_none() {
                    if let Ok(value) = input.try(|i| transition_property::SingleSpecifiedValue::parse(i)) {
                        property = Some(Some(value));
                        continue;
                    } else if input.try(|i| i.expect_ident_matching("none")).is_ok() {
                        // 'none' is not a valid value for <single-transition-property>,
                        // so it's not acceptable in the function above.
                        property = Some(None);
                        continue;
                    }
                }

                parsed -= 1;
                break
            }

            if parsed != 0 {
                Ok(SingleTransition {
                    % for prop in "duration timing_function delay".split():
                    transition_${prop}: ${prop}.unwrap_or_else(transition_${prop}::single_value
                                                                                ::get_initial_specified_value),
                    % endfor
                    transition_property: property.unwrap_or(
                        Some(transition_property::single_value::get_initial_specified_value())),
                })
            } else {
                Err(StyleParseError::UnspecifiedError.into())
            }
        }

        % for prop in "property duration timing_function delay".split():
        let mut ${prop}s = Vec::new();
        % endfor

        let results = input.parse_comma_separated(|i| parse_one_transition(context, i))?;
        let multiple_items = results.len() >= 2;
        for result in results {
            if let Some(value) = result.transition_property {
                propertys.push(value);
            } else if multiple_items {
                // If there is more than one item, and any of transitions has 'none',
                // then it's invalid. Othersize, leave propertys to be empty (which
                // means "transition-property: none");
                return Err(StyleParseError::UnspecifiedError.into());
            }

            % for prop in "duration timing_function delay".split():
            ${prop}s.push(result.transition_${prop});
            % endfor
        }

        Ok(expanded! {
            % for prop in "property duration timing_function delay".split():
            transition_${prop}: transition_${prop}::SpecifiedValue(${prop}s),
            % endfor
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            let property_len = self.transition_property.0.len();

            // There are two cases that we can do shorthand serialization:
            // * when all value lists have the same length, or
            // * when transition-property is none, and other value lists have exactly one item.
            if property_len == 0 {
                % for name in "duration delay timing_function".split():
                    if self.transition_${name}.0.len() != 1 {
                        return Ok(());
                    }
                % endfor
            } else {
                % for name in "duration delay timing_function".split():
                    if self.transition_${name}.0.len() != property_len {
                        return Ok(());
                    }
                % endfor
            }

            // Representative length.
            let len = self.transition_duration.0.len();

            for i in 0..len {
                if i != 0 {
                    dest.write_str(", ")?;
                }
                if property_len == 0 {
                    dest.write_str("none")?;
                } else {
                    self.transition_property.0[i].to_css(dest)?;
                }
                % for name in "duration timing_function delay".split():
                    dest.write_str(" ")?;
                    self.transition_${name}.0[i].to_css(dest)?;
                % endfor
            }
            Ok(())
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="animation" extra_prefixes="moz webkit"
                    sub_properties="animation-name animation-duration
                                    animation-timing-function animation-delay
                                    animation-iteration-count animation-direction
                                    animation-fill-mode animation-play-state"
                    allowed_in_keyframe_block="False"
                    spec="https://drafts.csswg.org/css-animations/#propdef-animation">
    <%
        props = "name duration timing_function delay iteration_count \
                 direction fill_mode play_state".split()
    %>
    use parser::Parse;
    % for prop in props:
    use properties::longhands::animation_${prop};
    % endfor

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        struct SingleAnimation {
            % for prop in props:
            animation_${prop}: animation_${prop}::SingleSpecifiedValue,
            % endfor
        }

        fn parse_one_animation<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                                       -> Result<SingleAnimation,ParseError<'i>> {
            % for prop in props:
            let mut ${prop} = None;
            % endfor

            let mut parsed = 0;
            // NB: Name must be the last one here so that keywords valid for other
            // longhands are not interpreted as names.
            //
            // Also, duration must be before delay, see
            // https://drafts.csswg.org/css-animations/#typedef-single-animation
            loop {
                parsed += 1;
                try_parse_one!(context, input, duration, animation_duration);
                try_parse_one!(context, input, timing_function, animation_timing_function);
                try_parse_one!(context, input, delay, animation_delay);
                try_parse_one!(context, input, iteration_count, animation_iteration_count);
                try_parse_one!(input, direction, animation_direction);
                try_parse_one!(input, fill_mode, animation_fill_mode);
                try_parse_one!(input, play_state, animation_play_state);
                try_parse_one!(context, input, name, animation_name);

                parsed -= 1;
                break
            }

            // If nothing is parsed, this is an invalid entry.
            if parsed == 0 {
                Err(StyleParseError::UnspecifiedError.into())
            } else {
                Ok(SingleAnimation {
                    % for prop in props:
                    animation_${prop}: ${prop}.unwrap_or_else(animation_${prop}::single_value
                                                              ::get_initial_specified_value),
                    % endfor
                })
            }
        }

        % for prop in props:
        let mut ${prop}s = vec![];
        % endfor

        let results = input.parse_comma_separated(|i| parse_one_animation(context, i))?;
        for result in results.into_iter() {
            % for prop in props:
            ${prop}s.push(result.animation_${prop});
            % endfor
        }

        Ok(expanded! {
            % for prop in props:
            animation_${prop}: animation_${prop}::SpecifiedValue(${prop}s),
            % endfor
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            let len = self.animation_name.0.len();
            // There should be at least one declared value
            if len == 0 {
                return Ok(());
            }

            // If any value list length is differs then we don't do a shorthand serialization
            // either.
            % for name in props[1:]:
                if len != self.animation_${name}.0.len() {
                    return Ok(())
                }
            % endfor

            for i in 0..len {
                if i != 0 {
                    dest.write_str(", ")?;
                }

                % for name in props[1:]:
                    self.animation_${name}.0[i].to_css(dest)?;
                    dest.write_str(" ")?;
                % endfor
                self.animation_name.0[i].to_css(dest)?;
            }
            Ok(())
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="scroll-snap-type" products="gecko"
                    sub_properties="scroll-snap-type-x scroll-snap-type-y"
                    spec="https://drafts.csswg.org/css-scroll-snap/#propdef-scroll-snap-type">
    use properties::longhands::scroll_snap_type_x;

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        let result = scroll_snap_type_x::parse(context, input)?;
        Ok(expanded! {
            scroll_snap_type_x: result,
            scroll_snap_type_y: result,
        })
    }

    impl<'a> ToCss for LonghandsToSerialize<'a>  {
        // Serializes into the single keyword value if both scroll-snap-type and scroll-snap-type-y are same.
        // Otherwise into an empty string. This is done to match Gecko's behaviour.
        fn to_css<W>(&self, dest: &mut W) -> fmt::Result where W: fmt::Write {
            if self.scroll_snap_type_x == self.scroll_snap_type_y {
                self.scroll_snap_type_x.to_css(dest)
            } else {
                Ok(())
            }
        }
    }
</%helpers:shorthand>

<%helpers:shorthand name="-moz-transform" products="gecko"
                    sub_properties="transform"
                    flags="SHORTHAND_ALIAS_PROPERTY"
                    derive_serialize="True"
                    spec="Non-standard: https://developer.mozilla.org/en-US/docs/Web/CSS/transform">
    use properties::longhands::transform;

    pub fn parse_value<'i, 't>(context: &ParserContext, input: &mut Parser<'i, 't>)
                               -> Result<Longhands, ParseError<'i>> {
        Ok(expanded! {
            transform: transform::parse_prefixed(context, input)?,
        })
    }
</%helpers:shorthand>
<|endoftext|>


<commit_before>//! Fully integrated benchmarks for rust-analyzer, which load real cargo
//! projects.
//!
//! The benchmark here is used to debug specific performance regressions. If you
//! notice that, eg, completion is slow in some specific case, you can  modify
//! code here exercise this specific completion, and thus have a fast
//! edit/compile/test cycle.
//!
//! Note that "Rust Analyzer: Run" action does not allow running a single test
//! in release mode in VS Code. There's however "Rust Analyzer: Copy Run Command Line"
//! which you can use to paste the command in terminal and add `--release` manually.

use std::{convert::TryFrom, sync::Arc};

use ide::{Change, CompletionConfig, FilePosition, TextSize};
use ide_db::helpers::{insert_use::InsertUseConfig, merge_imports::MergeBehavior, SnippetCap};
use test_utils::project_root;
use vfs::{AbsPathBuf, VfsPath};

use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig};

#[test]
fn integrated_highlighting_benchmark() {
    if std::env::var("RUN_SLOW_BENCHES").is_err() {
        return;
    }

    // Load rust-analyzer itself.
    let workspace_to_load = project_root();
    let file = "./crates/ide_db/src/apply_change.rs";

    let cargo_config = Default::default();
    let load_cargo_config = LoadCargoConfig {
        load_out_dirs_from_check: true,
        wrap_rustc: false,
        with_proc_macro: false,
    };

    let (mut host, vfs, _proc_macro) = {
        let _it = stdx::timeit("workspace loading");
        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
    };

    let file_id = {
        let file = workspace_to_load.join(file);
        let path = VfsPath::from(AbsPathBuf::assert(file));
        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
    };

    {
        let _it = stdx::timeit("initial");
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }

    profile::init_from("*>100");
    // let _s = profile::heartbeat_span();

    {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        text.push_str("\npub fn _dummy() {}\n");
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
    }

    {
        let _it = stdx::timeit("after change");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }
}

#[test]
fn integrated_completion_benchmark() {
    if std::env::var("RUN_SLOW_BENCHES").is_err() {
        return;
    }

    // Load rust-analyzer itself.
    let workspace_to_load = project_root();
    let file = "./crates/hir/src/lib.rs";

    let cargo_config = Default::default();
    let load_cargo_config = LoadCargoConfig {
        load_out_dirs_from_check: true,
        wrap_rustc: false,
        with_proc_macro: false,
    };

    let (mut host, vfs, _proc_macro) = {
        let _it = stdx::timeit("workspace loading");
        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
    };

    let file_id = {
        let file = workspace_to_load.join(file);
        let path = VfsPath::from(AbsPathBuf::assert(file));
        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
    };

    {
        let _it = stdx::timeit("initial");
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }

    profile::init_from("*>5");
    // let _s = profile::heartbeat_span();

    let completion_offset = {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        let completion_offset =
            patch(&mut text, "db.struct_data(self.id)", "sel;\ndb.struct_data(self.id)")
                + "sel".len();
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
        completion_offset
    };

    {
        let _it = stdx::timeit("unqualified path completion");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        let config = CompletionConfig {
            enable_postfix_completions: true,
            enable_imports_on_the_fly: true,
            add_call_parenthesis: true,
            add_call_argument_snippets: true,
            snippet_cap: SnippetCap::new(true),
            insert_use: InsertUseConfig {
                merge: Some(MergeBehavior::Crate),
                prefix_kind: hir::PrefixKind::ByCrate,
                group: true,
            },
        };
        let position =
            FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
        analysis.completions(&config, position).unwrap();
    }

    let completion_offset = {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        let completion_offset =
            patch(&mut text, "sel;\ndb.struct_data(self.id)", "self.;\ndb.struct_data(self.id)")
                + "self.".len();
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
        completion_offset
    };

    {
        let _it = stdx::timeit("dot completion");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        let config = CompletionConfig {
            enable_postfix_completions: true,
            enable_imports_on_the_fly: true,
            add_call_parenthesis: true,
            add_call_argument_snippets: true,
            snippet_cap: SnippetCap::new(true),
            insert_use: InsertUseConfig {
                merge: Some(MergeBehavior::Crate),
                prefix_kind: hir::PrefixKind::ByCrate,
                group: true,
            },
        };
        let position =
            FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
        analysis.completions(&config, position).unwrap();
    }
}

fn patch(what: &mut String, from: &str, to: &str) -> usize {
    let idx = what.find(from).unwrap();
    *what = what.replacen(from, to, 1);
    idx
}
<commit_msg>minor: more useful bench<commit_after>//! Fully integrated benchmarks for rust-analyzer, which load real cargo
//! projects.
//!
//! The benchmark here is used to debug specific performance regressions. If you
//! notice that, eg, completion is slow in some specific case, you can  modify
//! code here exercise this specific completion, and thus have a fast
//! edit/compile/test cycle.
//!
//! Note that "Rust Analyzer: Run" action does not allow running a single test
//! in release mode in VS Code. There's however "Rust Analyzer: Copy Run Command Line"
//! which you can use to paste the command in terminal and add `--release` manually.

use std::{convert::TryFrom, sync::Arc};

use ide::{Change, CompletionConfig, FilePosition, TextSize};
use ide_db::helpers::{insert_use::InsertUseConfig, merge_imports::MergeBehavior, SnippetCap};
use test_utils::project_root;
use vfs::{AbsPathBuf, VfsPath};

use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig};

#[test]
fn integrated_highlighting_benchmark() {
    if std::env::var("RUN_SLOW_BENCHES").is_err() {
        return;
    }

    // Load rust-analyzer itself.
    let workspace_to_load = project_root();
    let file = "./crates/ide_db/src/apply_change.rs";

    let cargo_config = Default::default();
    let load_cargo_config = LoadCargoConfig {
        load_out_dirs_from_check: true,
        wrap_rustc: false,
        with_proc_macro: false,
    };

    let (mut host, vfs, _proc_macro) = {
        let _it = stdx::timeit("workspace loading");
        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
    };

    let file_id = {
        let file = workspace_to_load.join(file);
        let path = VfsPath::from(AbsPathBuf::assert(file));
        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
    };

    {
        let _it = stdx::timeit("initial");
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }

    profile::init_from("*>100");
    // let _s = profile::heartbeat_span();

    {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        text.push_str("\npub fn _dummy() {}\n");
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
    }

    {
        let _it = stdx::timeit("after change");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }
}

#[test]
fn integrated_completion_benchmark() {
    if std::env::var("RUN_SLOW_BENCHES").is_err() {
        return;
    }

    // Load rust-analyzer itself.
    let workspace_to_load = project_root();
    let file = "./crates/hir/src/lib.rs";

    let cargo_config = Default::default();
    let load_cargo_config = LoadCargoConfig {
        load_out_dirs_from_check: true,
        wrap_rustc: false,
        with_proc_macro: false,
    };

    let (mut host, vfs, _proc_macro) = {
        let _it = stdx::timeit("workspace loading");
        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
    };

    let file_id = {
        let file = workspace_to_load.join(file);
        let path = VfsPath::from(AbsPathBuf::assert(file));
        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
    };

    {
        let _it = stdx::timeit("initial");
        let analysis = host.analysis();
        analysis.highlight_as_html(file_id, false).unwrap();
    }

    profile::init_from("*>5");
    // let _s = profile::heartbeat_span();

    let completion_offset = {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        let completion_offset =
            patch(&mut text, "db.struct_data(self.id)", "sel;\ndb.struct_data(self.id)")
                + "sel".len();
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
        completion_offset
    };

    {
        let _p = profile::span("unqualified path completion");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        let config = CompletionConfig {
            enable_postfix_completions: true,
            enable_imports_on_the_fly: true,
            add_call_parenthesis: true,
            add_call_argument_snippets: true,
            snippet_cap: SnippetCap::new(true),
            insert_use: InsertUseConfig {
                merge: Some(MergeBehavior::Crate),
                prefix_kind: hir::PrefixKind::ByCrate,
                group: true,
            },
        };
        let position =
            FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
        analysis.completions(&config, position).unwrap();
    }

    let completion_offset = {
        let _it = stdx::timeit("change");
        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
        let completion_offset =
            patch(&mut text, "sel;\ndb.struct_data(self.id)", "self.;\ndb.struct_data(self.id)")
                + "self.".len();
        let mut change = Change::new();
        change.change_file(file_id, Some(Arc::new(text)));
        host.apply_change(change);
        completion_offset
    };

    {
        let _p = profile::span("dot completion");
        let _span = profile::cpu_span();
        let analysis = host.analysis();
        let config = CompletionConfig {
            enable_postfix_completions: true,
            enable_imports_on_the_fly: true,
            add_call_parenthesis: true,
            add_call_argument_snippets: true,
            snippet_cap: SnippetCap::new(true),
            insert_use: InsertUseConfig {
                merge: Some(MergeBehavior::Crate),
                prefix_kind: hir::PrefixKind::ByCrate,
                group: true,
            },
        };
        let position =
            FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
        analysis.completions(&config, position).unwrap();
    }
}

fn patch(what: &mut String, from: &str, to: &str) -> usize {
    let idx = what.find(from).unwrap();
    *what = what.replacen(from, to, 1);
    idx
}
<|endoftext|>


<commit_before>#![feature(proc_macro_hygiene)]
#![feature(decl_macro)]
#![feature(rustc_private)]
#[macro_use] extern crate rocket;
#[macro_use] extern crate diesel;
extern crate serde_json;
extern crate serde;
extern crate dotenv;

use rocket::request::{Outcome, FromRequest};
use rocket::Outcome::Success;
use rocket::Request;

use rocket::Data;
use rocket::data::{self, FromDataSimple};
use rocket::http::Status;
use rocket::Outcome::Failure;
use serde::{Deserialize, Serialize};

use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use std::env;

use rocket::http::ContentType;

pub mod schema;
use schema::posts;

use std::rc::Rc;

#[derive(Debug)]
#[derive(Queryable)]
pub struct Post {
    pub id: i32,
    pub title: String,
    pub body: String,
    pub publihed: bool,
}

#[derive(Serialize, Deserialize)]
#[derive(Insertable)]
#[derive(Clone)]
#[derive(Debug)]
#[table_name="posts"]
pub struct NewPost{
    pub title: String,
    pub body: String,
}

impl FromDataSimple for NewPost {
    type Error = String;

    #[allow(unused_variables)]
    fn from_data(req: &Request, data: Data) -> data::Outcome<Self, String> {
        let reader = data.open();
        match serde_json::from_reader(reader).map(|val| val) {
            Ok(value) => Success(value),
            Err(e) => Failure((Status::BadRequest, e.to_string())),
        }
    }
}

// DB classes
pub struct DBPost {}

impl DBPost {
    // DATABASE classes
        pub fn establish_connection(self) -> PgConnection {
            dotenv().ok();
            let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
            PgConnection::establish(&database_url).expect(&format!("Error connecting to {}", database_url))
        }

        pub fn create(self, post: NewPost) -> Post {
            let conn = self.establish_connection();
            diesel::insert_into(posts::table).values(&post).get_result(&conn).expect("Error saving!")
        }
        pub fn read(self) -> Vec<Post> {
            let conn = self.establish_connection();
            posts::table.load::<Post>(&conn).unwrap()
      }
}

impl<'a, 'r> FromRequest<'a, 'r> for DBPost {
    type Error = ();
    fn from_request(_request: &'a Request<'r>) -> Outcome<Self, Self::Error> {
        Success(DBPost{})
    }
}

// Entities classes
pub trait UseCase {
    fn run(&self) -> &'static str;
}

struct HelloWorldCase {
    db: Rc<Option<DBPost>>
}

impl HelloWorldCase {
    pub fn new() -> HelloWorldCase {
        HelloWorldCase{db: Rc::new(None)}
    }
}

impl UseCase for HelloWorldCase {
    fn run(&self) -> &'static str {
        //unwrap shared reference
        let values = Rc::try_unwrap(Rc::clone(&self.db));
        let result = match values.unwrap_or(None)  {
            Some(unwrapped_db) => unwrapped_db.read(),
            None => Vec::new(),
        };
        println!("{:?}",result);
        "Hello world"
    }
}

struct AddNewPostCase {
    db: Rc<Option<DBPost>>,
    post: NewPost
}
impl AddNewPostCase  {
    pub fn new(post: NewPost) -> AddNewPostCase {
        AddNewPostCase{db: Rc::new(None), post: post}
    }
}

impl UseCase for AddNewPostCase {
    fn run(&self) -> &'static str {
        //unwrap shared reference
        let unwrap_db = Rc::try_unwrap(Rc::clone(&self.db));
        let created_post = match unwrap_db {
            Ok(v) => Some(v.unwrap().create(self.post.clone())),
            Err(_) => {println!("It was not possible ost the result"); None},
        };
        "Hello world"
    }
}

impl<'a, 'r> FromRequest<'a, 'r> for HelloWorldCase {
    type Error = ();
    fn from_request(_request: &'a Request<'r>) -> Outcome<Self, Self::Error> {
        Success(HelloWorldCase::new())
    }
}

#[get("/")]
fn get(db: DBPost) -> &'static str {
    HelloWorldCase{db: Rc::new(Some(db))}.run()
}
#[post("/", format="application/json", data="<post>")]
fn post(db: DBPost, post: NewPost) -> &'static str {
    println!("{:?}", post);
    AddNewPostCase{db: Rc::new(Some(db)), post:post}.run()
}

fn main() {
    let db = DBPost{};
    rocket::ignite().manage(db).mount("/", routes![get, post]).launch();
}

#[test]
fn test1() {
    use rocket::local::Client;
//    let use_case = HelloWorldCase{};
    let db = DBPost{};
    let rocket = rocket::ignite().manage(db).mount("/", routes![get, post]);
    let client = Client::new(rocket).unwrap();
    let mut response = client.get("/").dispatch();
    assert_eq!(response.body_string(), Some("Hello world".into()));
}

#[test]
fn test2() {
    use rocket::local::Client;
//    let use_case = HelloWorldCase{};
    let db = DBPost{};
    let rocket = rocket::ignite().manage(db).mount("/", routes![get, post]);
    let client = Client::new(rocket).unwrap();
    let mut response = client.post("/").header(ContentType::JSON).body("{\"title\": \"mytitle1\", \"body\": \"mybody1\"}").dispatch();
    assert_eq!(response.body_string(), Some("Hello world".into()));
}
<commit_msg>Added connection to db working<commit_after>#![feature(proc_macro_hygiene)]
#![feature(decl_macro)]
#![feature(rustc_private)]
#[macro_use] extern crate rocket;
#[macro_use] extern crate diesel;
extern crate serde_json;
extern crate serde;
extern crate dotenv;

use rocket::request::{Outcome, FromRequest};
use rocket::Outcome::Success;
use rocket::Request;

use rocket::Data;
use rocket::data::{self, FromDataSimple};
use rocket::http::Status;
use rocket::Outcome::Failure;
use serde::{Deserialize, Serialize};

use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use std::env;

use rocket::http::ContentType;

pub mod schema;
use schema::posts;

use std::rc::Rc;

#[derive(Debug)]
#[derive(Queryable)]
pub struct Post {
    pub id: i32,
    pub title: String,
    pub body: String,
    pub publihed: bool,
}

#[derive(Serialize, Deserialize)]
#[derive(Insertable)]
#[derive(Clone)]
#[derive(Debug)]
#[table_name="posts"]
pub struct NewPost{
    pub title: String,
    pub body: String,
}

impl FromDataSimple for NewPost {
    type Error = String;

    #[allow(unused_variables)]
    fn from_data(req: &Request, data: Data) -> data::Outcome<Self, String> {
        let reader = data.open();
        match serde_json::from_reader(reader).map(|val| val) {
            Ok(value) => Success(value),
            Err(e) => Failure((Status::BadRequest, e.to_string())),
        }
    }
}

// DB classes
pub struct DBPost {}

impl DBPost {
    // DATABASE classes
        pub fn establish_connection(self) -> PgConnection {
            dotenv().ok();
            let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
            PgConnection::establish(&database_url).expect(&format!("Error connecting to {}", database_url))
        }

        pub fn create(self, post: NewPost) -> Post {
            let conn = self.establish_connection();
            diesel::insert_into(posts::table).values(&post).get_result(&conn).expect("Error saving!")
        }
        pub fn read(self) -> Vec<Post> {
            let conn = self.establish_connection();
            posts::table.load::<Post>(&conn).unwrap()
      }
}

impl<'a, 'r> FromRequest<'a, 'r> for DBPost {
    type Error = ();
    fn from_request(_request: &'a Request<'r>) -> Outcome<Self, Self::Error> {
        Success(DBPost{})
    }
}

// Entities classes
pub trait UseCase {
    fn run(&self) -> &'static str;
}

struct HelloWorldCase {
    db: Rc<Option<DBPost>>
}

impl HelloWorldCase {
    pub fn new() -> HelloWorldCase {
        HelloWorldCase{db: Rc::new(None)}
    }
}

impl UseCase for HelloWorldCase {
    fn run(&self) -> &'static str {
        //unwrap shared reference
        /*
        let values = Rc::try_unwrap(Rc::clone(&self.db));
        let result = match values.unwrap_or(None)  {
            Some(unwrapped_db) => unwrapped_db.read(),
            None => Vec::new(),
        };
        */
        let db = DBPost{};
        let result = db.read();
        println!("{:?}",result);
        "Hello world"
    }
}

struct AddNewPostCase {
    db: Rc<Option<DBPost>>,
    post: NewPost
}
impl AddNewPostCase  {
    pub fn new(post: NewPost) -> AddNewPostCase {
        AddNewPostCase{db: Rc::new(None), post: post}
    }
}

impl UseCase for AddNewPostCase {
    fn run(&self) -> &'static str {
        //unwrap shared reference
        /*
        let unwrap_db = Rc::try_unwrap(Rc::clone(&self.db));
        let created_post = match unwrap_db {
            Ok(v) => Some(v.unwrap().create(self.post.clone())),
            Err(_) => {println!("It was not possible ost the result"); None},
        };
        */
        //TODO find best way to do it... added generics?
        let db = DBPost{};
        let result = db.create(self.post.clone());
        "Hello world"
    }
}

impl<'a, 'r> FromRequest<'a, 'r> for HelloWorldCase {
    type Error = ();
    fn from_request(_request: &'a Request<'r>) -> Outcome<Self, Self::Error> {
        Success(HelloWorldCase::new())
    }
}

#[get("/")]
fn get(db: DBPost) -> &'static str {
    HelloWorldCase{db: Rc::new(Some(db))}.run()
}
#[post("/", format="application/json", data="<post>")]
fn post(db: DBPost, post: NewPost) -> &'static str {
    println!("{:?}", post);
    AddNewPostCase{db: Rc::new(Some(db)), post:post}.run()
}

fn main() {
    let db = DBPost{};
    rocket::ignite().manage(db).mount("/", routes![get, post]).launch();
}

#[test]
fn test1() {
    use rocket::local::Client;
//    let use_case = HelloWorldCase{};
    let db = DBPost{};
    let rocket = rocket::ignite().manage(db).mount("/", routes![get, post]);
    let client = Client::new(rocket).unwrap();
    let mut response = client.get("/").dispatch();
    assert_eq!(response.body_string(), Some("Hello world".into()));
}

#[test]
fn test2() {
    use rocket::local::Client;
//    let use_case = HelloWorldCase{};
    let db = DBPost{};
    let rocket = rocket::ignite().manage(db).mount("/", routes![get, post]);
    let client = Client::new(rocket).unwrap();
    let mut response = client.post("/").header(ContentType::JSON).body("{\"title\": \"mytitle1\", \"body\": \"mybody1\"}").dispatch();
    assert_eq!(response.body_string(), Some("Hello world".into()));
}
<|endoftext|>


<commit_before>extern crate core;

use std::mem;
use self::core::str::next_code_point;
use super::token::Token;
use super::token::Category;

pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store
/// tokens for the various language and format lexers.
pub struct Tokenizer {
    data: Vec<u8>,
    head: usize,
    tail: usize,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.to_string().into_bytes(),
      head: 0,
      tail: 0,
      tokens: vec![],
      states: vec![]
    }
}

impl Tokenizer {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// let tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    /// tokenizer.advance();
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        if self.has_more_data() {
            // Get the current character so that we can calculate
            // its byte length and advance the head appropriately.
            match self.current_char() {
                Some(c) => {
                    self.head += c.len_utf8();
                },
                None => (),
            }
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("l");
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    /// tokenizer.advance();
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        if self.has_more_data() {
            // Create an iterator for the remaining data.
            let mut remaining_data = self.data[self.head..].iter();

            // Pull the first UTF-8 byte sequence and convert it to a char.
            next_code_point(&mut remaining_data).map(|code_point| {
                unsafe {
                    mem::transmute(code_point)
                }
            })
        } else {
            None
        }
    }

    /// Whether or not the remaining data starts
    /// with the specified string.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// assert!(tokenizer.starts_with("luth"));
    /// tokenizer.advance();
    /// assert!(tokenizer.starts_with("utho"));
    /// assert!(!tokenizer.starts_with("luth"));
    /// ```
    pub fn starts_with(&self, data: &str) -> bool {
        // Get a byte representation of the passed data
        // that we'll compare to the binary buffer.
        let data_bytes = data.as_bytes();

        // Don't even bother if the remaining data is smaller
        // than the string we're going to compare it to.
        let remaining_data_size = self.data.len() - self.head;
        if remaining_data_size >= data_bytes.len() {
            // There's enough data left. Take a leading slice of the same size for comparison.
            let leading_data = &self.data[self.head..(self.head+data_bytes.len())];

            // Compare!
            leading_data == data_bytes
        } else {
            // Not enough remaining data.
            false
        }
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if self.head > self.tail {
            // Build the lexeme by slicing the currently
            // selected range out of the buffer.
            let lexeme = unsafe {
                String::from_utf8_unchecked(
                    self.data[self.tail..self.head].to_vec()
                )
            };
            let token = Token{
                lexeme: lexeme,
                category: category,
            };
            self.tokens.push(token);
            self.tail = self.head;
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// Category::Text category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    /// assert_eq!(tokenizer.tokens()[0], Token{ lexeme: "l".to_string(), category: Category::Text});
    /// assert_eq!(tokenizer.tokens()[1], Token{ lexeme: "uthor".to_string(), category: Category::Keyword});
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }

    /// Determines whether or not there is more unprocessed data.
    fn has_more_data(&self) -> bool {
        self.head < self.data.len()
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    // Benchmarking
    extern crate test;
    use self::test::Bencher;

    #[test]
    fn advance_does_nothing_when_there_is_no_more_data() {
        let data = "élégant";
        let mut tokenizer = new(data);

        // Try to go beyond the last character.
        for _ in 0..10 {
            tokenizer.advance();
        }

        assert!(!tokenizer.has_more_data())
    }

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[bench]
    fn bench_current_char(b: &mut Bencher) {
        let data = include_str!("../test_data/data.json");
        let mut tokenizer = new(data);
        b.iter(|| tokenizer.current_char());
    }

    #[bench]
    fn bench_starts_with(b: &mut Bencher) {
        let data = include_str!("../test_data/data.json");
        let mut tokenizer = new(data);
        b.iter(|| tokenizer.starts_with("something"));
    }
}
<commit_msg>Implement tokenizer using an iterator-based approach.<commit_after>extern crate core;

use std::str::Chars;
use super::token::Token;
use super::token::Category;

pub struct StateFunction(pub fn(&mut Tokenizer) -> Option<StateFunction>);

/// The Tokenizer type is used to produce and store
/// tokens for the various language and format lexers.
pub struct Tokenizer<'a> {
    data: Chars<'a>,
    current_token: String,
    tokens: Vec<Token>,
    pub states: Vec<StateFunction>,
}

/// Initializes a new tokenizer with the given data.
///
/// # Examples
///
/// ```
/// let tokenizer = luthor::tokenizer::new("luthor");
/// ```
pub fn new(data: &str) -> Tokenizer {
    Tokenizer{
      data: data.chars(),
      current_token: String::new(),
      tokens: vec![],
      states: vec![]
    }
}

impl<'a> Tokenizer<'a> {
    /// Returns a copy of the tokens processed to date.
    ///
    /// # Examples
    ///
    /// ```
    /// let tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.tokens();
    /// ```
    pub fn tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Moves to the next character in the data.
    /// Does nothing if there is no more data to process.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    /// tokenizer.advance();
    /// assert_eq!(tokenizer.current_char().unwrap(), 'u');
    /// ```
    pub fn advance(&mut self) {
        match self.data.next() {
            Some(c) => self.current_token.push(c),
            None => ()
        }
    }

    /// Returns the character at the current position,
    /// unless all of the data has been processed.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("l");
    /// assert_eq!(tokenizer.current_char().unwrap(), 'l');
    /// tokenizer.advance();
    /// assert_eq!(tokenizer.current_char(), None);
    /// ```
    pub fn current_char(&self) -> Option<char> {
        match self.data.clone().peekable().peek() {
            Some(c) => Some(c.clone()),
            None => None
        }
    }

    /// Whether or not the remaining data starts
    /// with the specified string.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// assert!(tokenizer.starts_with("luth"));
    /// tokenizer.advance();
    /// assert!(tokenizer.starts_with("utho"));
    /// assert!(!tokenizer.starts_with("luth"));
    /// ```
    pub fn starts_with(&self, data: &str) -> bool {
        // Take a segment of the same size.
        let segment: String = self.data.clone().take(data.chars().count()).collect();
        segment == data
    }

    /// Creates and stores a token with the given category containing any
    /// data processed using `advance` since the last call to this method.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.advance();
    /// tokenizer.advance();
    /// tokenizer.tokenize(Category::Text);
    /// assert_eq!(tokenizer.tokens()[0].lexeme, "lu");
    /// ```
    pub fn tokenize(&mut self, category: Category) {
        if !self.current_token.is_empty() {
            let token = Token{
                lexeme: self.current_token.clone(),
                category: category,
            };
            self.tokens.push(token);
            self.current_token = String::new();
        }
    }

    /// Creates and stores a token with the given category and the
    /// next `amount` characters of the data. Before doing this, it
    /// tokenizes any previously processed characters with the generic
    /// Category::Text category.
    ///
    /// # Examples
    ///
    /// ```
    /// use luthor::token::Category;
    /// use luthor::token::Token;
    ///
    /// let mut tokenizer = luthor::tokenizer::new("luthor");
    /// tokenizer.advance();
    /// tokenizer.tokenize_next(5, Category::Keyword);
    /// assert_eq!(tokenizer.tokens()[0], Token{ lexeme: "l".to_string(), category: Category::Text});
    /// assert_eq!(tokenizer.tokens()[1], Token{ lexeme: "uthor".to_string(), category: Category::Keyword});
    /// ```
    pub fn tokenize_next(&mut self, amount: usize, category: Category) {
        // If there's any data that has yet
        // to be tokenized, take care of that.
        self.tokenize(Category::Text);

        // Mark the next series of characters.
        for _ in 0..amount { self.advance(); }

        // Tokenize the marked characters.
        self.tokenize(category);
    }
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::super::token::Token;
    use super::super::token::Category;

    // Benchmarking
    extern crate test;
    use self::test::Bencher;

    #[test]
    fn current_char_returns_the_char_at_head() {
        let data = "él";
        let tokenizer = new(data);

        assert_eq!(tokenizer.current_char().unwrap(), 'é');
    }

    #[test]
    fn current_char_returns_none_if_at_the_end() {
        let data = "él";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();

        assert_eq!(tokenizer.current_char(), None);
    }

    #[test]
    fn tokenize_creates_the_correct_token() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize(Category::Text);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_does_nothing_if_range_is_empty() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.tokenize(Category::Text);

        assert_eq!(tokenizer.tokens.len(), 0);
    }

    #[test]
    fn tokenize_next_tokenizes_previous_data_as_text() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(1, Category::Keyword);

        let token = tokenizer.tokens.remove(0);
        let expected_token = Token{ lexeme: "él".to_string(), category: Category::Text};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_tokenizes_next_x_chars() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(5, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[test]
    fn tokenize_next_takes_at_most_what_is_left() {
        let data = "élégant";
        let mut tokenizer = new(data);
        tokenizer.advance();
        tokenizer.advance();
        tokenizer.tokenize_next(15, Category::Keyword);

        let token = tokenizer.tokens.pop().unwrap();
        let expected_token = Token{ lexeme: "égant".to_string(), category: Category::Keyword};
        assert_eq!(token, expected_token);
    }

    #[bench]
    fn bench_current_char(b: &mut Bencher) {
        let data = include_str!("../test_data/data.json");
        let mut tokenizer = new(data);
        b.iter(|| tokenizer.current_char());
    }

    #[bench]
    fn bench_starts_with(b: &mut Bencher) {
        let data = include_str!("../test_data/data.json");
        let mut tokenizer = new(data);
        b.iter(|| tokenizer.starts_with("something"));
    }
}
<|endoftext|>