<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*!
The representation of CSS property values

Enums are named after the property. Variants have the same name + the
name of the value used in the spec. This leads to some verbose names,
e.g.:

The property 'background-color' and the specified value called '<color>'
in the spec lead to the variant CSSBackgroundColorColor(Color).

At least it's consistent though.
*/

use std::cmp::Eq;
use extra::url::Url;
use units::{Length, AbsoluteSize, RelativeSize};
use units::GenericFontFamily;
use color::Color;

/** A partial CSS value, before inheritance has been resolved */
#[deriving(Eq)]
pub enum CSSValue<T> {
    Inherit,
    Specified(T),
}

// CSS 2.1, Section 8 - Box model

#[deriving(Eq)]
pub enum CSSMargin {
    CSSMarginLength(Length),
    CSSMarginPercentage(float),
    CSSMarginAuto
}

#[deriving(Eq)]
pub enum CSSPadding {
    CSSPaddingLength(Length),
    CSSPaddingPercentage(float)
}

#[deriving(Eq)]
pub enum CSSBorderWidth {
    CSSBorderWidthThin,
    CSSBorderWidthMedium,
    CSSBorderWidthThick,
    CSSBorderWidthLength(Length)
}

#[deriving(Eq)]
pub enum CSSBorderColor {
    CSSBorderColorColor(Color),
    CSSBorderColorTransparent
}

#[deriving(Eq, Clone)]
pub enum CSSBorderStyle {
    CSSBorderStyleNone,
    CSSBorderStyleHidden,
    CSSBorderStyleDotted,
    CSSBorderStyleDashed,
    CSSBorderStyleSolid,
    CSSBorderStyleDouble,
    CSSBorderStyleGroove,
    CSSBorderStyleRidge,
    CSSBorderStyleInset,
    CSSBorderStyleOutset,
}

// CSS 2.1, Section 9 - Visual formatting model

#[deriving(Eq)]
pub enum CSSDisplay {
    CSSDisplayInline,
    CSSDisplayBlock,
    CSSDisplayListItem,
    CSSDisplayInlineBlock,
    CSSDisplayTable,
    CSSDisplayInlineTable,
    CSSDisplayTableRowGroup,
    CSSDisplayTableHeaderGroup,
    CSSDisplayTableFooterGroup,
    CSSDisplayTableRow,
    CSSDisplayTableColumnGroup,
    CSSDisplayTableColumn,
    CSSDisplayTableCell,
    CSSDisplayTableCaption,
    CSSDisplayNone
}

#[deriving(Eq)]
pub enum CSSPosition {
    CSSPositionStatic,
    CSSPositionRelative,
    CSSPositionAbsolute,
    CSSPositionFixed
}

#[deriving(Eq)]
pub enum CSSTop {
    CSSTopLength(Length),
    CSSTopPercentage,
    CSSTopAuto
}

#[deriving(Eq)]
pub enum CSSRight {
    CSSRightLength(Length),
    CSSRightPercentage(float),
    CSSRightAuto
}

#[deriving(Eq)]
pub enum CSSBottom {
    CSSBottomLength(Length),
    CSSBottomPercentage(float),
    CSSBottomAuto
}

#[deriving(Eq)]
pub enum CSSLeft {
    CSSLeftLength(Length),
    CSSLeftPercentage(float),
    CSSLeftAuto
}

#[deriving(Eq)]
pub enum CSSFloat {
    CSSFloatLeft,
    CSSFloatRight,
    CSSFloatNone
}

#[deriving(Eq)]
pub enum CSSClear {
    CSSClearLeft,
    CSSClearRight,
    CSSClearBoth,
    CSSClearNone
}

#[deriving(Eq)]
pub enum CSSDirection {
    CSSDirectionLtr,
    CSSDirectionRtl
}

// CSS 2.1, Section 10 - Visual formatting model details

#[deriving(Eq)]
pub enum CSSWidth {
    CSSWidthLength(Length),
    CSSWidthPercentage(float),
    CSSWidthAuto
}

#[deriving(Eq)]
pub enum CSSHeight {
    CSSHeightLength(Length),
    CSSHeightPercentage(float),
    CSSHeightAuto
}

#[deriving(Eq)]
pub enum CSSLineHeight {
    CSSLineHeightNormal,
    CSSLineHeightNumber(float),
    CSSLineHeightLength(Length),
    CSSLineHeightPercentage(float),
}

#[deriving(Eq)]
pub enum CSSVerticalAlign {
    CSSVerticalAlignBaseline,
    CSSVerticalAlignSub,
    CSSVerticalAlignSuper,
    CSSVerticalAlignTop,
    CSSVerticalAlignTextTop,
    CSSVerticalAlignMiddle,
    CSSVerticalAlignBottom,
    CSSVerticalAlignTextBottom,
    CSSVerticalAlignPercentage(float),
    CSSVerticalAlignLength(Length),
}

// CSS 2.1, Section 11 - Visual effects

#[deriving(Eq)]
pub enum CSSOverflow {
    CSSOverflowVisible,
    CSSOverflowHidden,
    CSSOverflowScroll,
    CSSOverflowAuto
}

#[deriving(Eq)]
pub enum CSSVisibility {
    CSSVisibilityVisible,
    CSSVisibilityHidden,
    CSSVisibilityCollapse
}

// CSS 2.1, Section 12 - Generated content, automatic numbering, and lists

// CSS 2.1, Section 13 - Paged media

// CSS 2.1, Section 14 - Colors and Backgrounds

#[deriving(Eq)]
pub enum CSSColor {
    CSSColorColor(Color)
}

#[deriving(Eq)]
pub enum CSSBackgroundColor {
    CSSBackgroundColorColor(Color),
    CSSBackgroundColorTransparent
}

#[deriving(Eq)]
pub enum CSSBackgroundImage {
    CSSBackgroundUri(Url),
    CSSBackgroundImageNone
}

#[deriving(Eq)]
pub enum CSSBackgroundRepeat {
    CSSBackgroundRepeatRepeat,
    CSSBackgroundRepeatRepeatX,
    CSSBackgroundRepeatRepeatY,
    CSSBackgroundRepeatNoRepeat
}

#[deriving(Eq)]
pub enum CSSBackgroundAttachment {
    CSSBackgroundAttachmentScroll,
    CSSBackgroundAttachmentFixed
}

#[deriving(Eq)]
pub enum CSSBackgroundPosition {
    CSSBackgroundPositionPercentage(float),
    CSSBackgroundPositionLength(Length),
    CSSBackgroundPositionLeft,
    CSSBackgroundPositionCenter,
    CSSBackgroundPositionRight,
    CSSBackgroundPositionTop,
    CSSBackgroundPositionBottom
}

// CSS 2.1, Section 15 - Fonts

#[deriving(Eq)]
pub enum CSSFontFamily {
    CSSFontFamilyFamilyName(~str),
    CSSFontFamilyGenericFamily(GenericFontFamily)
}

#[deriving(Eq)]
pub enum CSSFontStyle {
    CSSFontStyleNormal,
    CSSFontStyleItalic,
    CSSFontStyleOblique
}

#[deriving(Eq)]
pub enum CSSFontWeight {
    CSSFontWeightNormal,
    CSSFontWeightBold,
    CSSFontWeightBolder,
    CSSFontWeightLighter,
    CSSFontWeight100,
    CSSFontWeight200,
    CSSFontWeight300,
    CSSFontWeight400,
    CSSFontWeight500,
    CSSFontWeight600,
    CSSFontWeight700,
    CSSFontWeight800,
    CSSFontWeight900
}

#[deriving(Eq)]
pub enum CSSFontSize {
    CSSFontSizeAbsoluteSize(AbsoluteSize),
    CSSFontSizeRelativeSize(RelativeSize),
    CSSFontSizeLength(Length),
    CSSFontSizePercentage(float)
}

// CSS 2.1, Section 16 - Text

#[deriving(Eq)]
pub enum CSSTextAlign {
    CSSTextAlignLeft,
    CSSTextAlignRight,
    CSSTextAlignCenter,
    CSSTextAlignJustify
}

#[deriving(Eq)]
pub enum CSSTextDecoration {
    CSSTextDecorationNone,
    CSSTextDecorationUnderline,
    CSSTextDecorationOverline,
    CSSTextDecorationLineThrough,
    CSSTextDecorationBlink
}

#[deriving(Eq)]
pub enum CSSTextTransform {
    CSSTextTransformCapitalize,
    CSSTextTransformUppercase,
    CSSTextTransformLowercase,
    CSSTextTransformNone
}

// CSS 2.1, Section 17 - Tables

// CSS 2.1, Section 18 - User interface

<commit_msg>Derive Clone for CSSTextDecoration<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*!
The representation of CSS property values

Enums are named after the property. Variants have the same name + the
name of the value used in the spec. This leads to some verbose names,
e.g.:

The property 'background-color' and the specified value called '<color>'
in the spec lead to the variant CSSBackgroundColorColor(Color).

At least it's consistent though.
*/

use std::cmp::Eq;
use extra::url::Url;
use units::{Length, AbsoluteSize, RelativeSize};
use units::GenericFontFamily;
use color::Color;

/** A partial CSS value, before inheritance has been resolved */
#[deriving(Eq)]
pub enum CSSValue<T> {
    Inherit,
    Specified(T),
}

// CSS 2.1, Section 8 - Box model

#[deriving(Eq)]
pub enum CSSMargin {
    CSSMarginLength(Length),
    CSSMarginPercentage(float),
    CSSMarginAuto
}

#[deriving(Eq)]
pub enum CSSPadding {
    CSSPaddingLength(Length),
    CSSPaddingPercentage(float)
}

#[deriving(Eq)]
pub enum CSSBorderWidth {
    CSSBorderWidthThin,
    CSSBorderWidthMedium,
    CSSBorderWidthThick,
    CSSBorderWidthLength(Length)
}

#[deriving(Eq)]
pub enum CSSBorderColor {
    CSSBorderColorColor(Color),
    CSSBorderColorTransparent
}

#[deriving(Eq, Clone)]
pub enum CSSBorderStyle {
    CSSBorderStyleNone,
    CSSBorderStyleHidden,
    CSSBorderStyleDotted,
    CSSBorderStyleDashed,
    CSSBorderStyleSolid,
    CSSBorderStyleDouble,
    CSSBorderStyleGroove,
    CSSBorderStyleRidge,
    CSSBorderStyleInset,
    CSSBorderStyleOutset,
}

// CSS 2.1, Section 9 - Visual formatting model

#[deriving(Eq)]
pub enum CSSDisplay {
    CSSDisplayInline,
    CSSDisplayBlock,
    CSSDisplayListItem,
    CSSDisplayInlineBlock,
    CSSDisplayTable,
    CSSDisplayInlineTable,
    CSSDisplayTableRowGroup,
    CSSDisplayTableHeaderGroup,
    CSSDisplayTableFooterGroup,
    CSSDisplayTableRow,
    CSSDisplayTableColumnGroup,
    CSSDisplayTableColumn,
    CSSDisplayTableCell,
    CSSDisplayTableCaption,
    CSSDisplayNone
}

#[deriving(Eq)]
pub enum CSSPosition {
    CSSPositionStatic,
    CSSPositionRelative,
    CSSPositionAbsolute,
    CSSPositionFixed
}

#[deriving(Eq)]
pub enum CSSTop {
    CSSTopLength(Length),
    CSSTopPercentage,
    CSSTopAuto
}

#[deriving(Eq)]
pub enum CSSRight {
    CSSRightLength(Length),
    CSSRightPercentage(float),
    CSSRightAuto
}

#[deriving(Eq)]
pub enum CSSBottom {
    CSSBottomLength(Length),
    CSSBottomPercentage(float),
    CSSBottomAuto
}

#[deriving(Eq)]
pub enum CSSLeft {
    CSSLeftLength(Length),
    CSSLeftPercentage(float),
    CSSLeftAuto
}

#[deriving(Eq)]
pub enum CSSFloat {
    CSSFloatLeft,
    CSSFloatRight,
    CSSFloatNone
}

#[deriving(Eq)]
pub enum CSSClear {
    CSSClearLeft,
    CSSClearRight,
    CSSClearBoth,
    CSSClearNone
}

#[deriving(Eq)]
pub enum CSSDirection {
    CSSDirectionLtr,
    CSSDirectionRtl
}

// CSS 2.1, Section 10 - Visual formatting model details

#[deriving(Eq)]
pub enum CSSWidth {
    CSSWidthLength(Length),
    CSSWidthPercentage(float),
    CSSWidthAuto
}

#[deriving(Eq)]
pub enum CSSHeight {
    CSSHeightLength(Length),
    CSSHeightPercentage(float),
    CSSHeightAuto
}

#[deriving(Eq)]
pub enum CSSLineHeight {
    CSSLineHeightNormal,
    CSSLineHeightNumber(float),
    CSSLineHeightLength(Length),
    CSSLineHeightPercentage(float),
}

#[deriving(Eq)]
pub enum CSSVerticalAlign {
    CSSVerticalAlignBaseline,
    CSSVerticalAlignSub,
    CSSVerticalAlignSuper,
    CSSVerticalAlignTop,
    CSSVerticalAlignTextTop,
    CSSVerticalAlignMiddle,
    CSSVerticalAlignBottom,
    CSSVerticalAlignTextBottom,
    CSSVerticalAlignPercentage(float),
    CSSVerticalAlignLength(Length),
}

// CSS 2.1, Section 11 - Visual effects

#[deriving(Eq)]
pub enum CSSOverflow {
    CSSOverflowVisible,
    CSSOverflowHidden,
    CSSOverflowScroll,
    CSSOverflowAuto
}

#[deriving(Eq)]
pub enum CSSVisibility {
    CSSVisibilityVisible,
    CSSVisibilityHidden,
    CSSVisibilityCollapse
}

// CSS 2.1, Section 12 - Generated content, automatic numbering, and lists

// CSS 2.1, Section 13 - Paged media

// CSS 2.1, Section 14 - Colors and Backgrounds

#[deriving(Eq)]
pub enum CSSColor {
    CSSColorColor(Color)
}

#[deriving(Eq)]
pub enum CSSBackgroundColor {
    CSSBackgroundColorColor(Color),
    CSSBackgroundColorTransparent
}

#[deriving(Eq)]
pub enum CSSBackgroundImage {
    CSSBackgroundUri(Url),
    CSSBackgroundImageNone
}

#[deriving(Eq)]
pub enum CSSBackgroundRepeat {
    CSSBackgroundRepeatRepeat,
    CSSBackgroundRepeatRepeatX,
    CSSBackgroundRepeatRepeatY,
    CSSBackgroundRepeatNoRepeat
}

#[deriving(Eq)]
pub enum CSSBackgroundAttachment {
    CSSBackgroundAttachmentScroll,
    CSSBackgroundAttachmentFixed
}

#[deriving(Eq)]
pub enum CSSBackgroundPosition {
    CSSBackgroundPositionPercentage(float),
    CSSBackgroundPositionLength(Length),
    CSSBackgroundPositionLeft,
    CSSBackgroundPositionCenter,
    CSSBackgroundPositionRight,
    CSSBackgroundPositionTop,
    CSSBackgroundPositionBottom
}

// CSS 2.1, Section 15 - Fonts

#[deriving(Eq)]
pub enum CSSFontFamily {
    CSSFontFamilyFamilyName(~str),
    CSSFontFamilyGenericFamily(GenericFontFamily)
}

#[deriving(Eq)]
pub enum CSSFontStyle {
    CSSFontStyleNormal,
    CSSFontStyleItalic,
    CSSFontStyleOblique
}

#[deriving(Eq)]
pub enum CSSFontWeight {
    CSSFontWeightNormal,
    CSSFontWeightBold,
    CSSFontWeightBolder,
    CSSFontWeightLighter,
    CSSFontWeight100,
    CSSFontWeight200,
    CSSFontWeight300,
    CSSFontWeight400,
    CSSFontWeight500,
    CSSFontWeight600,
    CSSFontWeight700,
    CSSFontWeight800,
    CSSFontWeight900
}

#[deriving(Eq)]
pub enum CSSFontSize {
    CSSFontSizeAbsoluteSize(AbsoluteSize),
    CSSFontSizeRelativeSize(RelativeSize),
    CSSFontSizeLength(Length),
    CSSFontSizePercentage(float)
}

// CSS 2.1, Section 16 - Text

#[deriving(Eq)]
pub enum CSSTextAlign {
    CSSTextAlignLeft,
    CSSTextAlignRight,
    CSSTextAlignCenter,
    CSSTextAlignJustify
}

#[deriving(Eq, Clone)]
pub enum CSSTextDecoration {
    CSSTextDecorationNone,
    CSSTextDecorationUnderline,
    CSSTextDecorationOverline,
    CSSTextDecorationLineThrough,
    CSSTextDecorationBlink
}

#[deriving(Eq)]
pub enum CSSTextTransform {
    CSSTextTransformCapitalize,
    CSSTextTransformUppercase,
    CSSTextTransformLowercase,
    CSSTextTransformNone
}

// CSS 2.1, Section 17 - Tables

// CSS 2.1, Section 18 - User interface

<|endoftext|>


<commit_before>//! Compiler plugin for Rust-PHF
//!
//! See the documentation for the `phf` crate for more details.
#![crate_name="phf_mac"]
#![crate_type="dylib"]
#![doc(html_root_url="http://sfackler.github.io/rust-phf/doc")]
#![feature(plugin_registrar, quote)]

extern crate rand;
extern crate syntax;
extern crate time;
extern crate phf;
extern crate rustc;

use std::collections::HashMap;
use std::gc::{Gc, GC};
use std::os;
use syntax::ast;
use syntax::ast::{TokenTree, LitStr, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{DummyResult,
                        ExtCtxt,
                        MacResult,
                        MacExpr};
use syntax::parse;
use syntax::parse::token::{InternedString, COMMA, EOF, FAT_ARROW};
use rand::{Rng, SeedableRng, XorShiftRng};
use rustc::plugin::Registry;

static DEFAULT_LAMBDA: uint = 5;

static FIXED_SEED: [u32, ..4] = [3141592653, 589793238, 462643383, 2795028841];

#[plugin_registrar]
#[doc(hidden)]
pub fn macro_registrar(reg: &mut Registry) {
    reg.register_macro("phf_map", expand_phf_map);
    reg.register_macro("phf_set", expand_phf_set);
    reg.register_macro("phf_ordered_map", expand_phf_ordered_map);
    reg.register_macro("phf_ordered_set", expand_phf_ordered_set);
}

struct Entry {
    key_str: InternedString,
    key: Gc<Expr>,
    value: Gc<Expr>
}

struct HashState {
    k1: u64,
    k2: u64,
    disps: Vec<(uint, uint)>,
    map: Vec<uint>,
}

fn expand_phf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                  -> Box<MacResult> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_map(cx, sp, entries, state)
}

fn expand_phf_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                  -> Box<MacResult> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_set(cx, sp, entries, state)
}

fn expand_phf_ordered_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                          -> Box<MacResult> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp),
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_ordered_map(cx, sp, entries, state)
}

fn expand_phf_ordered_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                          -> Box<MacResult> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_ordered_set(cx, sp, entries, state)
}

fn parse_map(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                Vec::from_slice(tts));
    let mut entries = Vec::new();

    let mut bad = false;
    while parser.token != EOF {
        let key = cx.expand_expr(parser.parse_expr());
        let key_str = parse_str(cx, key).unwrap_or_else(|| {
            bad = true;
            InternedString::new("")
        });

        if !parser.eat(&FAT_ARROW) {
            cx.span_err(parser.span, "expected `=>`");
            return None;
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_str: key_str,
            key: key,
            value: value
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} entries are not supported",
                            phf::MAX_SIZE).as_slice());
        return None;
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_set(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                Vec::from_slice(tts));
    let mut entries = Vec::new();
    let value = quote_expr!(&*cx, ());

    let mut bad = false;
    while parser.token != EOF {
        let key = cx.expand_expr(parser.parse_expr());
        let key_str = parse_str(cx, key).unwrap_or_else(|| {
            bad = true;
            InternedString::new("")
        });

        entries.push(Entry {
            key_str: key_str,
            key: key,
            value: value,
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} entries are not supported",
                            phf::MAX_SIZE).as_slice());
        return None;
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_str(cx: &mut ExtCtxt, e: &Expr) -> Option<InternedString> {
    match e.node {
        ExprLit(lit) => {
            match lit.node {
                LitStr(ref s, _) => Some(s.clone()),
                _ => {
                    cx.span_err(e.span, "expected string literal");
                    None
                }
            }
        }
        _ => {
            cx.span_err(e.span, "expected string literal");
            None
        }
    }
}

fn has_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut strings = HashMap::new();
    for entry in entries.iter() {
        let spans = strings.find_or_insert(entry.key_str.clone(), vec![]);
        spans.push(entry.key.span);
    }

    for (key, spans) in strings.iter() {
        if spans.len() == 1 {
            continue;
        }

        dups = true;
        cx.span_err(sp,
                format!("duplicate key `{}`", key).as_slice());
        for span in spans.iter() {
            cx.span_note(*span, "one occurrence here");
        }
    }

    dups
}

fn generate_hash(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> HashState {
    let mut rng: XorShiftRng = SeedableRng::from_seed(FIXED_SEED);
    let start = time::precise_time_s();
    let state;
    loop {
        match try_generate_hash(entries.as_slice(), &mut rng) {
            Some(s) => {
                state = s;
                break;
            }
            None => {}
        }
    }
    let time = time::precise_time_s() - start;
    if os::getenv("PHF_STATS").is_some() {
        cx.span_note(sp, format!("PHF generation took {} seconds", time)
                            .as_slice());
    }

    state
}

fn try_generate_hash(entries: &[Entry], rng: &mut XorShiftRng)
                     -> Option<HashState> {
    struct Bucket {
        idx: uint,
        keys: Vec<uint>,
    }

    struct Hashes {
        g: uint,
        f1: uint,
        f2: uint,
    }

    let k1 = rng.gen();
    let k2 = rng.gen();

    let hashes: Vec<Hashes> = entries.iter().map(|entry| {
        let (g, f1, f2) = phf::hash(&entry.key_str.get(), k1, k2);
        Hashes {
            g: g,
            f1: f1,
            f2: f2
        }
    }).collect();

    let buckets_len = (entries.len() + DEFAULT_LAMBDA - 1) / DEFAULT_LAMBDA;
    let mut buckets = Vec::from_fn(buckets_len,
                                   |i| Bucket { idx: i, keys: Vec::new() });

    for (i, hash) in hashes.iter().enumerate() {
        buckets.get_mut(hash.g % buckets_len).keys.push(i);
    }

    // Sort descending
    buckets.sort_by(|a, b| b.keys.len().cmp(&a.keys.len()));

    let table_len = entries.len();
    let mut map = Vec::from_elem(table_len, None);
    let mut disps = Vec::from_elem(buckets_len, (0u, 0u));
    let mut try_map = HashMap::new();
    'buckets: for bucket in buckets.iter() {
        for d1 in range(0, table_len) {
            'disps_l: for d2 in range(0, table_len) {
                try_map.clear();
                for &key in bucket.keys.iter() {
                    let idx = phf::displace(hashes.get(key).f1,
                                            hashes.get(key).f2,
                                            d1,
                                            d2) % table_len;
                    if map.get(idx).is_some() || try_map.find(&idx).is_some() {
                        continue 'disps_l;
                    }
                    try_map.insert(idx, key);
                }

                // We've picked a good set of disps
                *disps.get_mut(bucket.idx) = (d1, d2);
                for (&idx, &key) in try_map.iter() {
                    *map.get_mut(idx) = Some(key);
                }
                continue 'buckets;
            }
        }

        // Unable to find displacements for a bucket
        return None;
    }

    Some(HashState {
        k1: k1,
        k2: k2,
        disps: disps,
        map: map.move_iter().map(|i| i.unwrap()).collect(),
    })
}

fn create_map(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>, state: HashState)
              -> Box<MacResult> {
    let disps = state.disps.iter().map(|&(d1, d2)| {
        quote_expr!(&*cx, ($d1, $d2))
    }).collect();
    let disps = create_slice_expr(disps, sp);

    let entries = state.map.iter().map(|&idx| {
        let &Entry { key, value, .. } = entries.get(idx);
        quote_expr!(&*cx, ($key, $value))
    }).collect();
    let entries = create_slice_expr(entries, sp);

    let k1 = state.k1;
    let k2 = state.k2;
    MacExpr::new(quote_expr!(cx, ::phf::PhfMap {
        k1: $k1,
        k2: $k2,
        disps: &$disps,
        entries: &$entries,
    }))
}

fn create_set(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>, state: HashState)
              -> Box<MacResult> {
    let map = create_map(cx, sp, entries, state).make_expr().unwrap();
    MacExpr::new(quote_expr!(cx, ::phf::PhfSet { map: $map }))
}

fn create_ordered_map(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>,
                      state: HashState) -> Box<MacResult> {
    let disps = state.disps.iter().map(|&(d1, d2)| {
        quote_expr!(&*cx, ($d1, $d2))
    }).collect();
    let disps = create_slice_expr(disps, sp);

    let idxs = state.map.iter().map(|&idx| quote_expr!(&*cx, $idx)).collect();
    let idxs = create_slice_expr(idxs, sp);

    let entries = entries.iter().map(|&Entry { key, value, .. }| {
        quote_expr!(&*cx, ($key, $value))
    }).collect();
    let entries = create_slice_expr(entries, sp);

    let k1 = state.k1;
    let k2 = state.k2;
    MacExpr::new(quote_expr!(cx, ::phf::PhfOrderedMap {
        k1: $k1,
        k2: $k2,
        disps: &$disps,
        idxs: &$idxs,
        entries: &$entries,
    }))
}

fn create_ordered_set(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>,
                      state: HashState) -> Box<MacResult> {
    let map = create_ordered_map(cx, sp, entries, state).make_expr().unwrap();
    MacExpr::new(quote_expr!(cx, ::phf::PhfOrderedSet { map: $map }))
}

fn create_slice_expr(vec: Vec<Gc<Expr>>, sp: Span) -> Gc<Expr> {
    box (GC) Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(vec),
        span: sp
    }
}
<commit_msg>Plugin infrastructure for non-str keys<commit_after>//! Compiler plugin for Rust-PHF
//!
//! See the documentation for the `phf` crate for more details.
#![crate_name="phf_mac"]
#![crate_type="dylib"]
#![doc(html_root_url="http://sfackler.github.io/rust-phf/doc")]
#![feature(plugin_registrar, quote, default_type_params)]

extern crate rand;
extern crate syntax;
extern crate time;
extern crate phf;
extern crate rustc;

use std::collections::HashMap;
use std::fmt;
use std::gc::{Gc, GC};
use std::hash;
use std::hash::{Hash};
use std::os;
use syntax::ast;
use syntax::ast::{TokenTree, LitStr, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{DummyResult,
                        ExtCtxt,
                        MacResult,
                        MacExpr};
use syntax::parse;
use syntax::parse::token::{InternedString, COMMA, EOF, FAT_ARROW};
use rand::{Rng, SeedableRng, XorShiftRng};
use rustc::plugin::Registry;

static DEFAULT_LAMBDA: uint = 5;

static FIXED_SEED: [u32, ..4] = [3141592653, 589793238, 462643383, 2795028841];

#[plugin_registrar]
#[doc(hidden)]
pub fn macro_registrar(reg: &mut Registry) {
    reg.register_macro("phf_map", expand_phf_map);
    reg.register_macro("phf_set", expand_phf_set);
    reg.register_macro("phf_ordered_map", expand_phf_ordered_map);
    reg.register_macro("phf_ordered_set", expand_phf_ordered_set);
}

#[deriving(PartialEq, Eq, Clone)]
enum Key {
    StaticStr(InternedString),
}

impl<S: hash::Writer> Hash<S> for Key {
    fn hash(&self, state: &mut S) {
        match *self {
            StaticStr(ref s) => s.get().hash(state),
        }
    }
}

impl fmt::Show for Key {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            StaticStr(ref s) => s.fmt(fmt),
        }
    }
}

struct Entry {
    key_contents: Key,
    key: Gc<Expr>,
    value: Gc<Expr>
}

struct HashState {
    k1: u64,
    k2: u64,
    disps: Vec<(uint, uint)>,
    map: Vec<uint>,
}

fn expand_phf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                  -> Box<MacResult> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_map(cx, sp, entries, state)
}

fn expand_phf_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                  -> Box<MacResult> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_set(cx, sp, entries, state)
}

fn expand_phf_ordered_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                          -> Box<MacResult> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp),
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_ordered_map(cx, sp, entries, state)
}

fn expand_phf_ordered_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                          -> Box<MacResult> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, entries.as_slice()) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, entries.as_slice());

    create_ordered_set(cx, sp, entries, state)
}

fn parse_map(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                Vec::from_slice(tts));
    let mut entries = Vec::new();

    let mut bad = false;
    while parser.token != EOF {
        let key = cx.expand_expr(parser.parse_expr());
        let key_contents = parse_key(cx, key).unwrap_or_else(|| {
            bad = true;
            StaticStr(InternedString::new(""))
        });

        if !parser.eat(&FAT_ARROW) {
            cx.span_err(parser.span, "expected `=>`");
            return None;
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} entries are not supported",
                            phf::MAX_SIZE).as_slice());
        return None;
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_set(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                Vec::from_slice(tts));
    let mut entries = Vec::new();
    let value = quote_expr!(&*cx, ());

    let mut bad = false;
    while parser.token != EOF {
        let key = cx.expand_expr(parser.parse_expr());
        let key_contents = parse_key(cx, key).unwrap_or_else(|| {
            bad = true;
            StaticStr(InternedString::new(""))
        });

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value,
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} entries are not supported",
                            phf::MAX_SIZE).as_slice());
        return None;
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_key(cx: &mut ExtCtxt, e: &Expr) -> Option<Key> {
    match e.node {
        ExprLit(lit) => {
            match lit.node {
                LitStr(ref s, _) => Some(StaticStr(s.clone())),
                _ => {
                    cx.span_err(e.span, "expected string literal");
                    None
                }
            }
        }
        _ => {
            cx.span_err(e.span, "expected string literal");
            None
        }
    }
}

fn has_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut strings = HashMap::new();
    for entry in entries.iter() {
        let spans = strings.find_or_insert(entry.key_contents.clone(), vec![]);
        spans.push(entry.key.span);
    }

    for (key, spans) in strings.iter() {
        if spans.len() == 1 {
            continue;
        }

        dups = true;
        cx.span_err(sp,
                format!("duplicate key `{}`", key).as_slice());
        for span in spans.iter() {
            cx.span_note(*span, "one occurrence here");
        }
    }

    dups
}

fn generate_hash(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> HashState {
    let mut rng: XorShiftRng = SeedableRng::from_seed(FIXED_SEED);
    let start = time::precise_time_s();
    let state;
    loop {
        match try_generate_hash(entries.as_slice(), &mut rng) {
            Some(s) => {
                state = s;
                break;
            }
            None => {}
        }
    }
    let time = time::precise_time_s() - start;
    if os::getenv("PHF_STATS").is_some() {
        cx.span_note(sp, format!("PHF generation took {} seconds", time)
                            .as_slice());
    }

    state
}

fn try_generate_hash(entries: &[Entry], rng: &mut XorShiftRng)
                     -> Option<HashState> {
    struct Bucket {
        idx: uint,
        keys: Vec<uint>,
    }

    struct Hashes {
        g: uint,
        f1: uint,
        f2: uint,
    }

    let k1 = rng.gen();
    let k2 = rng.gen();

    let hashes: Vec<Hashes> = entries.iter().map(|entry| {
        let (g, f1, f2) = phf::hash(&entry.key_contents, k1, k2);
        Hashes {
            g: g,
            f1: f1,
            f2: f2
        }
    }).collect();

    let buckets_len = (entries.len() + DEFAULT_LAMBDA - 1) / DEFAULT_LAMBDA;
    let mut buckets = Vec::from_fn(buckets_len,
                                   |i| Bucket { idx: i, keys: Vec::new() });

    for (i, hash) in hashes.iter().enumerate() {
        buckets.get_mut(hash.g % buckets_len).keys.push(i);
    }

    // Sort descending
    buckets.sort_by(|a, b| b.keys.len().cmp(&a.keys.len()));

    let table_len = entries.len();
    let mut map = Vec::from_elem(table_len, None);
    let mut disps = Vec::from_elem(buckets_len, (0u, 0u));
    let mut try_map = HashMap::new();
    'buckets: for bucket in buckets.iter() {
        for d1 in range(0, table_len) {
            'disps_l: for d2 in range(0, table_len) {
                try_map.clear();
                for &key in bucket.keys.iter() {
                    let idx = phf::displace(hashes.get(key).f1,
                                            hashes.get(key).f2,
                                            d1,
                                            d2) % table_len;
                    if map.get(idx).is_some() || try_map.find(&idx).is_some() {
                        continue 'disps_l;
                    }
                    try_map.insert(idx, key);
                }

                // We've picked a good set of disps
                *disps.get_mut(bucket.idx) = (d1, d2);
                for (&idx, &key) in try_map.iter() {
                    *map.get_mut(idx) = Some(key);
                }
                continue 'buckets;
            }
        }

        // Unable to find displacements for a bucket
        return None;
    }

    Some(HashState {
        k1: k1,
        k2: k2,
        disps: disps,
        map: map.move_iter().map(|i| i.unwrap()).collect(),
    })
}

fn create_map(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>, state: HashState)
              -> Box<MacResult> {
    let disps = state.disps.iter().map(|&(d1, d2)| {
        quote_expr!(&*cx, ($d1, $d2))
    }).collect();
    let disps = create_slice_expr(disps, sp);

    let entries = state.map.iter().map(|&idx| {
        let &Entry { key, value, .. } = entries.get(idx);
        quote_expr!(&*cx, ($key, $value))
    }).collect();
    let entries = create_slice_expr(entries, sp);

    let k1 = state.k1;
    let k2 = state.k2;
    MacExpr::new(quote_expr!(cx, ::phf::PhfMap {
        k1: $k1,
        k2: $k2,
        disps: &$disps,
        entries: &$entries,
    }))
}

fn create_set(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>, state: HashState)
              -> Box<MacResult> {
    let map = create_map(cx, sp, entries, state).make_expr().unwrap();
    MacExpr::new(quote_expr!(cx, ::phf::PhfSet { map: $map }))
}

fn create_ordered_map(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>,
                      state: HashState) -> Box<MacResult> {
    let disps = state.disps.iter().map(|&(d1, d2)| {
        quote_expr!(&*cx, ($d1, $d2))
    }).collect();
    let disps = create_slice_expr(disps, sp);

    let idxs = state.map.iter().map(|&idx| quote_expr!(&*cx, $idx)).collect();
    let idxs = create_slice_expr(idxs, sp);

    let entries = entries.iter().map(|&Entry { key, value, .. }| {
        quote_expr!(&*cx, ($key, $value))
    }).collect();
    let entries = create_slice_expr(entries, sp);

    let k1 = state.k1;
    let k2 = state.k2;
    MacExpr::new(quote_expr!(cx, ::phf::PhfOrderedMap {
        k1: $k1,
        k2: $k2,
        disps: &$disps,
        idxs: &$idxs,
        entries: &$entries,
    }))
}

fn create_ordered_set(cx: &mut ExtCtxt, sp: Span, entries: Vec<Entry>,
                      state: HashState) -> Box<MacResult> {
    let map = create_ordered_map(cx, sp, entries, state).make_expr().unwrap();
    MacExpr::new(quote_expr!(cx, ::phf::PhfOrderedSet { map: $map }))
}

fn create_slice_expr(vec: Vec<Gc<Expr>>, sp: Span) -> Gc<Expr> {
    box (GC) Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(vec),
        span: sp
    }
}
<|endoftext|>


<commit_before>// Write Markdown to the terminal

use crate::term2::{color, Attr, Terminal};
use std::io;

use pulldown_cmark::{Event, Tag};

// Handles the wrapping of text written to the console
struct LineWrapper<'a, T: Terminal> {
    indent: u32,
    margin: u32,
    pos: u32,
    pub w: &'a mut T,
}

impl<'a, T: Terminal + 'a> LineWrapper<'a, T> {
    // Just write a newline
    fn write_line(&mut self) {
        let _ = writeln!(self.w);
        // Reset column position to start of line
        self.pos = 0;
    }
    // Called before writing text to ensure indent is applied
    fn write_indent(&mut self) {
        if self.pos == 0 {
            // Write a space for each level of indent
            for _ in 0..self.indent {
                let _ = write!(self.w, " ");
            }
            self.pos = self.indent;
        }
    }
    // Write a non-breaking word
    fn write_word(&mut self, word: &str) {
        // Ensure correct indentation
        self.write_indent();
        let word_len = word.len() as u32;

        // If this word goes past the margin
        if self.pos + word_len > self.margin {
            // And adding a newline would give us more space
            if self.pos > self.indent {
                // Then add a newline!
                self.write_line();
                self.write_indent();
            }
        }

        // Write the word
        let _ = write!(self.w, "{}", word);
        self.pos += word_len;
    }
    fn write_space(&mut self) {
        if self.pos > self.indent {
            if self.pos < self.margin {
                self.write_word(" ");
            } else {
                self.write_line();
            }
        }
    }
    // Writes a span of text which wraps at the margin
    fn write_span(&mut self, text: &str) {
        // Allow words to wrap on whitespace
        let mut is_first = true;
        for word in text.split(char::is_whitespace) {
            if is_first {
                is_first = false;
            } else {
                self.write_space();
            }
            self.write_word(word);
        }
    }
    // Constructor
    fn new(w: &'a mut T, indent: u32, margin: u32) -> Self {
        LineWrapper {
            indent,
            margin,
            pos: indent,
            w,
        }
    }
}

// Handles the formatting of text
struct LineFormatter<'a, T: Terminal + io::Write> {
    is_code_block: bool,
    wrapper: LineWrapper<'a, T>,
    attrs: Vec<Attr>,
}

impl<'a, T: Terminal + io::Write + 'a> LineFormatter<'a, T> {
    fn new(w: &'a mut T, indent: u32, margin: u32) -> Self {
        LineFormatter {
            is_code_block: false,
            wrapper: LineWrapper::new(w, indent, margin),
            attrs: Vec::new(),
        }
    }
    fn push_attr(&mut self, attr: Attr) {
        self.attrs.push(attr);
        let _ = self.wrapper.w.attr(attr);
    }
    fn pop_attr(&mut self) {
        self.attrs.pop();
        let _ = self.wrapper.w.reset();
        for attr in &self.attrs {
            let _ = self.wrapper.w.attr(*attr);
        }
    }

    fn start_tag(&mut self, tag: Tag<'a>) {
        match tag {
            Tag::Paragraph => {
                self.wrapper.write_line();
            }
            Tag::Heading(_level) => {
                self.push_attr(Attr::Bold);
                self.wrapper.write_line();
            }
            Tag::Table(_alignments) => {}
            Tag::TableHead => {}
            Tag::TableRow => {}
            Tag::TableCell => {}
            Tag::BlockQuote => {}
            Tag::CodeBlock(_lang) => {
                self.wrapper.write_line();
                self.wrapper.indent += 2;
                self.is_code_block = true;
            }
            Tag::List(_) => {
                self.wrapper.write_line();
                self.wrapper.indent += 2;
            }
            Tag::Item => {
                self.wrapper.write_line();
            }
            Tag::Emphasis => {
                self.push_attr(Attr::ForegroundColor(color::RED));
            }
            Tag::Strong => {}
            Tag::Strikethrough => {}
            Tag::Link(_link_type, _dest, _title) => {}
            Tag::Image(_link_type, _dest, _title) => {}
            Tag::FootnoteDefinition(_name) => {}
        }
    }

    fn end_tag(&mut self, tag: Tag<'a>) {
        match tag {
            Tag::Paragraph => {
                self.wrapper.write_line();
            }
            Tag::Heading(_level) => {
                self.wrapper.write_line();
                self.pop_attr();
            }
            Tag::Table(_) => {}
            Tag::TableHead => {}
            Tag::TableRow => {}
            Tag::TableCell => {}
            Tag::BlockQuote => {}
            Tag::CodeBlock(_) => {
                self.is_code_block = false;
                self.wrapper.indent -= 2;
            }
            Tag::List(_) => {
                self.wrapper.indent -= 2;
                self.wrapper.write_line();
            }
            Tag::Item => {}
            Tag::Emphasis => {
                self.pop_attr();
            }
            Tag::Strong => {}
            Tag::Strikethrough => {}
            Tag::Link(_, _, _) => {}
            Tag::Image(_, _, _) => {} // shouldn't happen, handled in start
            Tag::FootnoteDefinition(_) => {}
        }
    }

    fn process_event(&mut self, event: Event<'a>) {
        use self::Event::*;
        match event {
            Start(tag) => self.start_tag(tag),
            End(tag) => self.end_tag(tag),
            Text(text) => {
                if self.is_code_block {
                    self.wrapper.write_word(&text);
                } else {
                    self.wrapper.write_span(&text);
                }
            }
            Code(code) => {
                self.push_attr(Attr::Bold);
                self.wrapper.write_word(&code);
                self.pop_attr();
            }
            Html(_html) => {}
            SoftBreak => {
                self.wrapper.write_line();
            }
            HardBreak => {
                self.wrapper.write_line();
            }
            Rule => {}
            FootnoteReference(_name) => {}
            TaskListMarker(true) => {}
            TaskListMarker(false) => {}
        }
    }
}

pub fn md<'a, S: AsRef<str>, T: Terminal + io::Write + 'a>(t: &'a mut T, content: S) {
    let mut f = LineFormatter::new(t, 0, 79);
    let parser = pulldown_cmark::Parser::new(content.as_ref());
    for event in parser {
        f.process_event(event);
    }
}
<commit_msg>markdown: Fix indentation of multi-line code blocks<commit_after>// Write Markdown to the terminal

use crate::term2::{color, Attr, Terminal};
use std::io;

use pulldown_cmark::{Event, Tag};

// Handles the wrapping of text written to the console
struct LineWrapper<'a, T: Terminal> {
    indent: u32,
    margin: u32,
    pos: u32,
    pub w: &'a mut T,
}

impl<'a, T: Terminal + 'a> LineWrapper<'a, T> {
    // Just write a newline
    fn write_line(&mut self) {
        let _ = writeln!(self.w);
        // Reset column position to start of line
        self.pos = 0;
    }
    // Called before writing text to ensure indent is applied
    fn write_indent(&mut self) {
        if self.pos == 0 {
            // Write a space for each level of indent
            for _ in 0..self.indent {
                let _ = write!(self.w, " ");
            }
            self.pos = self.indent;
        }
    }
    // Write a non-breaking word
    fn write_word(&mut self, word: &str) {
        // Ensure correct indentation
        self.write_indent();
        let word_len = word.len() as u32;

        // If this word goes past the margin
        if self.pos + word_len > self.margin {
            // And adding a newline would give us more space
            if self.pos > self.indent {
                // Then add a newline!
                self.write_line();
                self.write_indent();
            }
        }

        // Write the word
        let _ = write!(self.w, "{}", word);
        self.pos += word_len;
    }
    fn write_space(&mut self) {
        if self.pos > self.indent {
            if self.pos < self.margin {
                self.write_word(" ");
            } else {
                self.write_line();
            }
        }
    }
    // Writes a span of text which wraps at the margin
    fn write_span(&mut self, text: &str) {
        // Allow words to wrap on whitespace
        let mut is_first = true;
        for word in text.split(char::is_whitespace) {
            if is_first {
                is_first = false;
            } else {
                self.write_space();
            }
            self.write_word(word);
        }
    }
    // Writes code block where each line is indented
    fn write_code_block(&mut self, text: &str) {
        for line in text.lines() {
            self.write_word(line); // Will call write_indent()
            self.write_line();
        }
    }
    // Constructor
    fn new(w: &'a mut T, indent: u32, margin: u32) -> Self {
        LineWrapper {
            indent,
            margin,
            pos: indent,
            w,
        }
    }
}

// Handles the formatting of text
struct LineFormatter<'a, T: Terminal + io::Write> {
    is_code_block: bool,
    wrapper: LineWrapper<'a, T>,
    attrs: Vec<Attr>,
}

impl<'a, T: Terminal + io::Write + 'a> LineFormatter<'a, T> {
    fn new(w: &'a mut T, indent: u32, margin: u32) -> Self {
        LineFormatter {
            is_code_block: false,
            wrapper: LineWrapper::new(w, indent, margin),
            attrs: Vec::new(),
        }
    }
    fn push_attr(&mut self, attr: Attr) {
        self.attrs.push(attr);
        let _ = self.wrapper.w.attr(attr);
    }
    fn pop_attr(&mut self) {
        self.attrs.pop();
        let _ = self.wrapper.w.reset();
        for attr in &self.attrs {
            let _ = self.wrapper.w.attr(*attr);
        }
    }

    fn start_tag(&mut self, tag: Tag<'a>) {
        match tag {
            Tag::Paragraph => {
                self.wrapper.write_line();
            }
            Tag::Heading(_level) => {
                self.push_attr(Attr::Bold);
                self.wrapper.write_line();
            }
            Tag::Table(_alignments) => {}
            Tag::TableHead => {}
            Tag::TableRow => {}
            Tag::TableCell => {}
            Tag::BlockQuote => {}
            Tag::CodeBlock(_lang) => {
                self.wrapper.write_line();
                self.wrapper.indent += 2;
                self.is_code_block = true;
            }
            Tag::List(_) => {
                self.wrapper.write_line();
                self.wrapper.indent += 2;
            }
            Tag::Item => {
                self.wrapper.write_line();
            }
            Tag::Emphasis => {
                self.push_attr(Attr::ForegroundColor(color::RED));
            }
            Tag::Strong => {}
            Tag::Strikethrough => {}
            Tag::Link(_link_type, _dest, _title) => {}
            Tag::Image(_link_type, _dest, _title) => {}
            Tag::FootnoteDefinition(_name) => {}
        }
    }

    fn end_tag(&mut self, tag: Tag<'a>) {
        match tag {
            Tag::Paragraph => {
                self.wrapper.write_line();
            }
            Tag::Heading(_level) => {
                self.wrapper.write_line();
                self.pop_attr();
            }
            Tag::Table(_) => {}
            Tag::TableHead => {}
            Tag::TableRow => {}
            Tag::TableCell => {}
            Tag::BlockQuote => {}
            Tag::CodeBlock(_) => {
                self.is_code_block = false;
                self.wrapper.indent -= 2;
            }
            Tag::List(_) => {
                self.wrapper.indent -= 2;
                self.wrapper.write_line();
            }
            Tag::Item => {}
            Tag::Emphasis => {
                self.pop_attr();
            }
            Tag::Strong => {}
            Tag::Strikethrough => {}
            Tag::Link(_, _, _) => {}
            Tag::Image(_, _, _) => {} // shouldn't happen, handled in start
            Tag::FootnoteDefinition(_) => {}
        }
    }

    fn process_event(&mut self, event: Event<'a>) {
        use self::Event::*;
        match event {
            Start(tag) => self.start_tag(tag),
            End(tag) => self.end_tag(tag),
            Text(text) => {
                if self.is_code_block {
                    self.wrapper.write_code_block(&text);
                } else {
                    self.wrapper.write_span(&text);
                }
            }
            Code(code) => {
                self.push_attr(Attr::Bold);
                self.wrapper.write_word(&code);
                self.pop_attr();
            }
            Html(_html) => {}
            SoftBreak => {
                self.wrapper.write_line();
            }
            HardBreak => {
                self.wrapper.write_line();
            }
            Rule => {}
            FootnoteReference(_name) => {}
            TaskListMarker(true) => {}
            TaskListMarker(false) => {}
        }
    }
}

pub fn md<'a, S: AsRef<str>, T: Terminal + io::Write + 'a>(t: &'a mut T, content: S) {
    let mut f = LineFormatter::new(t, 0, 79);
    let parser = pulldown_cmark::Parser::new(content.as_ref());
    for event in parser {
        f.process_event(event);
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::error::{Error};
use std::fmt::{Display, Formatter};

#[derive(Debug)]
pub enum ReadError {
    Input(std::io::Error),
    Decode(prost::DecodeError),
    Magic(u32),
}

#[derive(Debug)]
pub enum WriteError {
    Output(std::io::Error),
    Encode(prost::EncodeError),
}

impl Display for ReadError {

    fn fmt(&self, fmt: &mut Formatter) -> std::fmt::Result {
        use ReadError::*;

        match *self {
            Input(ref err) => write!(fmt, "input error: {}", err),
            Decode(ref err) => write!(fmt, "proto decoding error: {}", err),
            Magic(magic) => write!(fmt, "invalid magic: {}", magic),
        }
    }
}

impl Display for WriteError {

    fn fmt(&self, fmt: &mut Formatter) -> std::fmt::Result {
        use WriteError::*;

        match *self {
            Output(ref err) => write!(fmt, "output error: {}", err),
            Encode(ref err) => write!(fmt, "proto encoding error: {}", err),
        }
    }
}

impl Error for ReadError {

    fn source(&self) -> Option<&(dyn Error + 'static)> {
        use ReadError::*;

        match *self {
            Input(ref err) => Some(err),
            Decode(ref err) => Some(err),
            Magic(_) => None,
        }
    }
}

impl Error for WriteError {

    fn source(&self) -> Option<&(dyn Error + 'static)> {
        use WriteError::*;

        match *self {
            Output(ref err) => Some(err),
            Encode(ref err) => Some(err),
        }
    }
}

impl From<std::io::Error> for ReadError {

    fn from(err: std::io::Error) -> ReadError {
        ReadError::Input(err)
    }
}

impl From<prost::DecodeError> for ReadError {

    fn from(err: prost::DecodeError) -> ReadError {
        ReadError::Decode(err)
    }
}

impl From<std::io::Error> for WriteError {

    fn from(err: std::io::Error) -> WriteError {
        WriteError::Output(err)
    }
}

impl From<prost::EncodeError> for WriteError {

    fn from(err: prost::EncodeError) -> WriteError {
        WriteError::Encode(err)
    }
}
<commit_msg>Add convenience conversions for the client errors.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::error::{Error};
use std::fmt::{Display, Formatter};

#[derive(Debug)]
pub enum ReadError {
    Input(std::io::Error),
    Decode(prost::DecodeError),
    Magic(u32),
}

#[derive(Debug)]
pub enum WriteError {
    Output(std::io::Error),
    Encode(prost::EncodeError),
}

impl Display for ReadError {

    fn fmt(&self, fmt: &mut Formatter) -> std::fmt::Result {
        use ReadError::*;

        match *self {
            Input(ref err) => write!(fmt, "input error: {}", err),
            Decode(ref err) => write!(fmt, "proto decoding error: {}", err),
            Magic(magic) => write!(fmt, "invalid magic: {}", magic),
        }
    }
}

impl Display for WriteError {

    fn fmt(&self, fmt: &mut Formatter) -> std::fmt::Result {
        use WriteError::*;

        match *self {
            Output(ref err) => write!(fmt, "output error: {}", err),
            Encode(ref err) => write!(fmt, "proto encoding error: {}", err),
        }
    }
}

impl Error for ReadError {

    fn source(&self) -> Option<&(dyn Error + 'static)> {
        use ReadError::*;

        match *self {
            Input(ref err) => Some(err),
            Decode(ref err) => Some(err),
            Magic(_) => None,
        }
    }
}

impl Error for WriteError {

    fn source(&self) -> Option<&(dyn Error + 'static)> {
        use WriteError::*;

        match *self {
            Output(ref err) => Some(err),
            Encode(ref err) => Some(err),
        }
    }
}

impl From<std::io::Error> for ReadError {

    fn from(err: std::io::Error) -> ReadError {
        ReadError::Input(err)
    }
}

impl From<prost::DecodeError> for ReadError {

    fn from(err: prost::DecodeError) -> ReadError {
        ReadError::Decode(err)
    }
}

impl From<std::io::Error> for WriteError {

    fn from(err: std::io::Error) -> WriteError {
        WriteError::Output(err)
    }
}

impl From<prost::EncodeError> for WriteError {

    fn from(err: prost::EncodeError) -> WriteError {
        WriteError::Encode(err)
    }
}

impl From<ReadError> for std::io::Error {

    fn from(err: ReadError) -> std::io::Error {
        use ReadError::*;

        match err {
            Input(err) => err,
            Decode(err) => err.into(),
            Magic(magic) => {
                let err = format!("invalid magic: {}", magic);
                std::io::Error::new(std::io::ErrorKind::InvalidData, err)
            },
        }
    }
}

impl From<WriteError> for std::io::Error {

    fn from(err: WriteError) -> std::io::Error {
        use WriteError::*;

        match err {
            Output(err) => err,
            Encode(err) => err.into(),
        }
    }
}
<|endoftext|>


<commit_before>//! Global `Arc`-based object interning infrastructure.
//!
//! Eventually this should probably be replaced with salsa-based interning.

use std::{
    fmt::{self, Debug},
    hash::{BuildHasherDefault, Hash},
    ops::Deref,
    sync::Arc,
};

use dashmap::{DashMap, SharedValue};
use once_cell::sync::OnceCell;
use rustc_hash::FxHasher;

type InternMap<T> = DashMap<Arc<T>, (), BuildHasherDefault<FxHasher>>;

#[derive(Hash)]
pub struct Interned<T: Internable> {
    arc: Arc<T>,
}

impl<T: Internable> Interned<T> {
    pub fn new(obj: T) -> Self {
        let storage = T::storage().get();
        let shard_idx = storage.determine_map(&obj);
        let shard = &storage.shards()[shard_idx];
        let shard = shard.upgradeable_read();

        // Atomically,
        // - check if `obj` is already in the map
        //   - if so, clone its `Arc` and return it
        //   - if not, box it up, insert it, and return a clone
        // This needs to be atomic (locking the shard) to avoid races with other thread, which could
        // insert the same object between us looking it up and inserting it.

        // FIXME: avoid double lookup by using raw entry API (once stable, or when hashbrown can be
        // plugged into dashmap)
        if let Some((arc, _)) = shard.get_key_value(&obj) {
            return Self { arc: arc.clone() };
        }

        let arc = Arc::new(obj);
        let arc2 = arc.clone();

        {
            let mut shard = shard.upgrade();
            shard.insert(arc2, SharedValue::new(()));
        }

        Self { arc }
    }
}

impl<T: Internable> Drop for Interned<T> {
    fn drop(&mut self) {
        // When the last `Ref` is dropped, remove the object from the global map.
        if Arc::strong_count(&self.arc) == 2 {
            // Only `self` and the global map point to the object.

            let storage = T::storage().get();
            let shard_idx = storage.determine_map(&self.arc);
            let shard = &storage.shards()[shard_idx];
            let mut shard = shard.write();

            // FIXME: avoid double lookup
            let (arc, _) =
                shard.get_key_value(&self.arc).expect("interned value removed prematurely");

            if Arc::strong_count(arc) != 2 {
                // Another thread has interned another copy
                return;
            }

            shard.remove(&self.arc);

            // Shrink the backing storage if the shard is less than 50% occupied.
            if shard.len() * 2 < shard.capacity() {
                shard.shrink_to_fit();
            }
        }
    }
}

/// Compares interned `Ref`s using pointer equality.
impl<T: Internable> PartialEq for Interned<T> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        Arc::ptr_eq(&self.arc, &other.arc)
    }
}

impl<T: Internable> Eq for Interned<T> {}

impl<T: Internable> AsRef<T> for Interned<T> {
    #[inline]
    fn as_ref(&self) -> &T {
        &self.arc
    }
}

impl<T: Internable> Deref for Interned<T> {
    type Target = T;

    #[inline]
    fn deref(&self) -> &Self::Target {
        &self.arc
    }
}

impl<T: Internable> Clone for Interned<T> {
    fn clone(&self) -> Self {
        Self { arc: self.arc.clone() }
    }
}

impl<T: Debug + Internable> Debug for Interned<T> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        (*self.arc).fmt(f)
    }
}

pub struct InternStorage<T> {
    map: OnceCell<InternMap<T>>,
}

impl<T> InternStorage<T> {
    pub const fn new() -> Self {
        Self { map: OnceCell::new() }
    }
}

impl<T: Internable> InternStorage<T> {
    fn get(&self) -> &InternMap<T> {
        self.map.get_or_init(DashMap::default)
    }
}

pub trait Internable: Hash + Eq + Sized + 'static {
    fn storage() -> &'static InternStorage<Self>;
}

// region:`Internable` implementations

macro_rules! impl_internable {
    ( $($t:ty),+ $(,)? ) => { $(
        impl Internable for $t {
            fn storage() -> &'static InternStorage<Self> {
                static STORAGE: InternStorage<$t> = InternStorage::new();
                &STORAGE
            }
        }
    )+ };
}

impl_internable!(crate::type_ref::TypeRef, crate::type_ref::TraitRef, crate::path::ModPath);

// endregion
<commit_msg>Remove unnecessary region, relax `Sized` bounds<commit_after>//! Global `Arc`-based object interning infrastructure.
//!
//! Eventually this should probably be replaced with salsa-based interning.

use std::{
    fmt::{self, Debug},
    hash::{BuildHasherDefault, Hash},
    ops::Deref,
    sync::Arc,
};

use dashmap::{DashMap, SharedValue};
use once_cell::sync::OnceCell;
use rustc_hash::FxHasher;

type InternMap<T> = DashMap<Arc<T>, (), BuildHasherDefault<FxHasher>>;

#[derive(Hash)]
pub struct Interned<T: Internable + ?Sized> {
    arc: Arc<T>,
}

impl<T: Internable> Interned<T> {
    pub fn new(obj: T) -> Self {
        let storage = T::storage().get();
        let shard_idx = storage.determine_map(&obj);
        let shard = &storage.shards()[shard_idx];
        let shard = shard.upgradeable_read();

        // Atomically,
        // - check if `obj` is already in the map
        //   - if so, clone its `Arc` and return it
        //   - if not, box it up, insert it, and return a clone
        // This needs to be atomic (locking the shard) to avoid races with other thread, which could
        // insert the same object between us looking it up and inserting it.

        // FIXME: avoid double lookup by using raw entry API (once stable, or when hashbrown can be
        // plugged into dashmap)
        if let Some((arc, _)) = shard.get_key_value(&obj) {
            return Self { arc: arc.clone() };
        }

        let arc = Arc::new(obj);
        let arc2 = arc.clone();

        {
            let mut shard = shard.upgrade();
            shard.insert(arc2, SharedValue::new(()));
        }

        Self { arc }
    }
}

impl<T: Internable + ?Sized> Drop for Interned<T> {
    fn drop(&mut self) {
        // When the last `Ref` is dropped, remove the object from the global map.
        if Arc::strong_count(&self.arc) == 2 {
            // Only `self` and the global map point to the object.

            let storage = T::storage().get();
            let shard_idx = storage.determine_map(&self.arc);
            let shard = &storage.shards()[shard_idx];
            let mut shard = shard.write();

            // FIXME: avoid double lookup
            let (arc, _) =
                shard.get_key_value(&self.arc).expect("interned value removed prematurely");

            if Arc::strong_count(arc) != 2 {
                // Another thread has interned another copy
                return;
            }

            shard.remove(&self.arc);

            // Shrink the backing storage if the shard is less than 50% occupied.
            if shard.len() * 2 < shard.capacity() {
                shard.shrink_to_fit();
            }
        }
    }
}

/// Compares interned `Ref`s using pointer equality.
impl<T: Internable + ?Sized> PartialEq for Interned<T> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        Arc::ptr_eq(&self.arc, &other.arc)
    }
}

impl<T: Internable + ?Sized> Eq for Interned<T> {}

impl<T: Internable + ?Sized> AsRef<T> for Interned<T> {
    #[inline]
    fn as_ref(&self) -> &T {
        &self.arc
    }
}

impl<T: Internable + ?Sized> Deref for Interned<T> {
    type Target = T;

    #[inline]
    fn deref(&self) -> &Self::Target {
        &self.arc
    }
}

impl<T: Internable + ?Sized> Clone for Interned<T> {
    fn clone(&self) -> Self {
        Self { arc: self.arc.clone() }
    }
}

impl<T: Debug + Internable + ?Sized> Debug for Interned<T> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        (*self.arc).fmt(f)
    }
}

pub struct InternStorage<T: ?Sized> {
    map: OnceCell<InternMap<T>>,
}

impl<T: ?Sized> InternStorage<T> {
    pub const fn new() -> Self {
        Self { map: OnceCell::new() }
    }
}

impl<T: Internable + ?Sized> InternStorage<T> {
    fn get(&self) -> &InternMap<T> {
        self.map.get_or_init(DashMap::default)
    }
}

pub trait Internable: Hash + Eq + 'static {
    fn storage() -> &'static InternStorage<Self>;
}

macro_rules! impl_internable {
    ( $($t:ty),+ $(,)? ) => { $(
        impl Internable for $t {
            fn storage() -> &'static InternStorage<Self> {
                static STORAGE: InternStorage<$t> = InternStorage::new();
                &STORAGE
            }
        }
    )+ };
}

impl_internable!(crate::type_ref::TypeRef, crate::type_ref::TraitRef, crate::path::ModPath);
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(plugin, rustc_private, str_char, collections)]

extern crate syntax;
extern crate rustc;

#[macro_use]
extern crate log;

use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufRead, Read};
use std::path::Path;

use syntax::parse;
use syntax::parse::lexer;
use rustc::session::{self, config};

use syntax::ast;
use syntax::ast::Name;
use syntax::codemap;
use syntax::codemap::Pos;
use syntax::parse::token;
use syntax::parse::lexer::TokenAndSpan;

fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
    fn id() -> token::Token {
        token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
    }

    let mut res = HashMap::new();

    res.insert("-1".to_string(), token::Eof);

    for line in file.split('\n') {
        let eq = match line.trim().rfind('=') {
            Some(val) => val,
            None => continue
        };

        let val = &line[..eq];
        let num = &line[eq + 1..];

        let tok = match val {
            "SHR"               => token::BinOp(token::Shr),
            "DOLLAR"            => token::Dollar,
            "LT"                => token::Lt,
            "STAR"              => token::BinOp(token::Star),
            "FLOAT_SUFFIX"      => id(),
            "INT_SUFFIX"        => id(),
            "SHL"               => token::BinOp(token::Shl),
            "LBRACE"            => token::OpenDelim(token::Brace),
            "RARROW"            => token::RArrow,
            "LIT_STR"           => token::Literal(token::Str_(Name(0)), None),
            "DOTDOT"            => token::DotDot,
            "MOD_SEP"           => token::ModSep,
            "DOTDOTDOT"         => token::DotDotDot,
            "NOT"               => token::Not,
            "AND"               => token::BinOp(token::And),
            "LPAREN"            => token::OpenDelim(token::Paren),
            "ANDAND"            => token::AndAnd,
            "AT"                => token::At,
            "LBRACKET"          => token::OpenDelim(token::Bracket),
            "LIT_STR_RAW"       => token::Literal(token::StrRaw(Name(0), 0), None),
            "RPAREN"            => token::CloseDelim(token::Paren),
            "SLASH"             => token::BinOp(token::Slash),
            "COMMA"             => token::Comma,
            "LIFETIME"          => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
            "CARET"             => token::BinOp(token::Caret),
            "TILDE"             => token::Tilde,
            "IDENT"             => id(),
            "PLUS"              => token::BinOp(token::Plus),
            "LIT_CHAR"          => token::Literal(token::Char(Name(0)), None),
            "LIT_BYTE"          => token::Literal(token::Byte(Name(0)), None),
            "EQ"                => token::Eq,
            "RBRACKET"          => token::CloseDelim(token::Bracket),
            "COMMENT"           => token::Comment,
            "DOC_COMMENT"       => token::DocComment(Name(0)),
            "DOT"               => token::Dot,
            "EQEQ"              => token::EqEq,
            "NE"                => token::Ne,
            "GE"                => token::Ge,
            "PERCENT"           => token::BinOp(token::Percent),
            "RBRACE"            => token::CloseDelim(token::Brace),
            "BINOP"             => token::BinOp(token::Plus),
            "POUND"             => token::Pound,
            "OROR"              => token::OrOr,
            "LIT_INTEGER"       => token::Literal(token::Integer(Name(0)), None),
            "BINOPEQ"           => token::BinOpEq(token::Plus),
            "LIT_FLOAT"         => token::Literal(token::Float(Name(0)), None),
            "WHITESPACE"        => token::Whitespace,
            "UNDERSCORE"        => token::Underscore,
            "MINUS"             => token::BinOp(token::Minus),
            "SEMI"              => token::Semi,
            "COLON"             => token::Colon,
            "FAT_ARROW"         => token::FatArrow,
            "OR"                => token::BinOp(token::Or),
            "GT"                => token::Gt,
            "LE"                => token::Le,
            "LIT_BINARY"        => token::Literal(token::Binary(Name(0)), None),
            "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0), None),
            "QUESTION"          => token::Question,
            "SHEBANG"           => token::Shebang(Name(0)),
            _                   => panic!("Bad token str `{}`", val),
        };

        res.insert(num.to_string(), tok);
    }

    debug!("Token map: {:?}", res);
    res
}

fn str_to_binop(s: &str) -> token::BinOpToken {
    match s {
        "+"     => token::Plus,
        "/"     => token::Slash,
        "-"     => token::Minus,
        "*"     => token::Star,
        "%"     => token::Percent,
        "^"     => token::Caret,
        "&"     => token::And,
        "|"     => token::Or,
        "<<"    => token::Shl,
        ">>"    => token::Shr,
        _       => panic!("Bad binop str `{}`", s),
    }
}

/// Assuming a string/binary literal, strip out the leading/trailing
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'r' {
        if lit.char_at(1) == 'b' {
            lit = &lit[2..]
        } else {
            lit = &lit[1..];
        }
    } else if lit.char_at(0) == 'b' {
        lit = &lit[1..];
    }

    let leading_hashes = count(lit);

    // +1/-1 to adjust for single quotes
    parse::token::intern(&lit[leading_hashes + 1..lit.len() - leading_hashes - 1])
}

/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'b' {
        lit = &lit[1..];
    }

    parse::token::intern(&lit[1..lit.len() - 1])
}

fn count(lit: &str) -> usize {
    lit.chars().take_while(|c| *c == '#').count()
}

fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize],
                     has_bom: bool)
                     -> TokenAndSpan {
    // old regex:
    // \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
    let start = s.find("[@").unwrap();
    let comma = start + s[start..].find(",").unwrap();
    let colon = comma + s[comma..].find(":").unwrap();
    let content_start = colon + s[colon..].find("='").unwrap();
    // Use rfind instead of find, because we don't want to stop at the content
    let content_end = content_start + s[content_start..].rfind("',<").unwrap();
    let toknum_end = content_end + s[content_end..].find(">,").unwrap();

    let start = &s[comma + 1 .. colon];
    let end = &s[colon + 1 .. content_start];
    let content = &s[content_start + 2 .. content_end];
    let toknum = &s[content_end + 3 .. toknum_end];

    let not_found = format!("didn't find token {:?} in the map", toknum);
    let proto_tok = tokens.get(toknum).expect(&not_found[..]);

    let nm = parse::token::intern(content);

    debug!("What we got: content (`{}`), proto: {:?}", content, proto_tok);

    let real_tok = match *proto_tok {
        token::BinOp(..)           => token::BinOp(str_to_binop(content)),
        token::BinOpEq(..)         => token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
        token::Literal(token::Str_(..), n)      => token::Literal(token::Str_(fix(content)), n),
        token::Literal(token::StrRaw(..), n)    => token::Literal(token::StrRaw(fix(content),
                                                                             count(content)), n),
        token::Literal(token::Char(..), n)      => token::Literal(token::Char(fixchar(content)), n),
        token::Literal(token::Byte(..), n)      => token::Literal(token::Byte(fixchar(content)), n),
        token::DocComment(..)      => token::DocComment(nm),
        token::Literal(token::Integer(..), n)   => token::Literal(token::Integer(nm), n),
        token::Literal(token::Float(..), n)     => token::Literal(token::Float(nm), n),
        token::Literal(token::Binary(..), n)    => token::Literal(token::Binary(nm), n),
        token::Literal(token::BinaryRaw(..), n) => token::Literal(token::BinaryRaw(fix(content),
                                                                                count(content)), n),
        token::Ident(..)           => token::Ident(ast::Ident { name: nm, ctxt: 0 },
                                                   token::ModName),
        token::Lifetime(..)        => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
        ref t => t.clone()
    };

    let start_offset = if real_tok == token::Eof {
        1
    } else {
        0
    };

    let offset = if has_bom { 1 } else { 0 };

    let mut lo = start.parse::<u32>().unwrap() - start_offset - offset;
    let mut hi = end.parse::<u32>().unwrap() + 1 - offset;

    // Adjust the span: For each surrogate pair already encountered, subtract one position.
    lo -= surrogate_pairs_pos.binary_search(&(lo as usize)).unwrap_or_else(|x| x) as u32;
    hi -= surrogate_pairs_pos.binary_search(&(hi as usize)).unwrap_or_else(|x| x) as u32;

    let sp = codemap::Span {
        lo: codemap::BytePos(lo),
        hi: codemap::BytePos(hi),
        expn_id: codemap::NO_EXPANSION
    };

    TokenAndSpan {
        tok: real_tok,
        sp: sp
    }
}

fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
    match a {
        &token::Ident(id, _) => match b {
                &token::Ident(id2, _) => id == id2,
                _ => false
        },
        _ => a == b
    }
}

fn span_cmp(antlr_sp: codemap::Span, rust_sp: codemap::Span, cm: &codemap::CodeMap) -> bool {
    antlr_sp.expn_id == rust_sp.expn_id &&
        antlr_sp.lo.to_usize() == cm.bytepos_to_file_charpos(rust_sp.lo).to_usize() &&
        antlr_sp.hi.to_usize() == cm.bytepos_to_file_charpos(rust_sp.hi).to_usize()
}

fn main() {
    fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
        use syntax::parse::lexer::Reader;
        r.next_token()
    }

    let mut args = env::args().skip(1);
    let filename = args.next().unwrap();
    if filename.find("parse-fail").is_some() {
        return;
    }

    // Rust's lexer
    let mut code = String::new();
    File::open(&Path::new(&filename)).unwrap().read_to_string(&mut code).unwrap();

    let surrogate_pairs_pos: Vec<usize> = code.chars().enumerate()
                                                     .filter(|&(_, c)| c as usize > 0xFFFF)
                                                     .map(|(n, _)| n)
                                                     .enumerate()
                                                     .map(|(x, n)| x + n)
                                                     .collect();

    let has_bom = code.starts_with("\u{feff}");

    debug!("Pairs: {:?}", surrogate_pairs_pos);

    let options = config::basic_options();
    let session = session::build_session(options, None,
                                         syntax::diagnostics::registry::Registry::new(&[]));
    let filemap = session.parse_sess.codemap().new_filemap(String::from_str("<n/a>"), code);
    let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
    let cm = session.codemap();

    // ANTLR
    let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();
    let mut token_list = String::new();
    token_file.read_to_string(&mut token_list).unwrap();
    let token_map = parse_token_list(&token_list[..]);

    let stdin = std::io::stdin();
    let lock = stdin.lock();
    let lines = lock.lines();
    let antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
                                                       &token_map,
                                                       &surrogate_pairs_pos[..],
                                                       has_bom));

    for antlr_tok in antlr_tokens {
        let rustc_tok = next(&mut lexer);
        if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
            continue
        }

        assert!(span_cmp(antlr_tok.sp, rustc_tok.sp, cm), "{:?} and {:?} have different spans",
                rustc_tok,
                antlr_tok);

        macro_rules! matches {
            ( $($x:pat),+ ) => (
                match rustc_tok.tok {
                    $($x => match antlr_tok.tok {
                        $x => {
                            if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
                                // FIXME #15677: needs more robust escaping in
                                // antlr
                                warn!("Different names for {:?} and {:?}", rustc_tok, antlr_tok);
                            }
                        }
                        _ => panic!("{:?} is not {:?}", antlr_tok, rustc_tok)
                    },)*
                    ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", antlr_tok, rustc_tok)
                }
            )
        }

        matches!(
            token::Literal(token::Byte(..), _),
            token::Literal(token::Char(..), _),
            token::Literal(token::Integer(..), _),
            token::Literal(token::Float(..), _),
            token::Literal(token::Str_(..), _),
            token::Literal(token::StrRaw(..), _),
            token::Literal(token::Binary(..), _),
            token::Literal(token::BinaryRaw(..), _),
            token::Ident(..),
            token::Lifetime(..),
            token::Interpolated(..),
            token::DocComment(..),
            token::Shebang(..)
        );
    }
}
<commit_msg>Rollup merge of #25537 - carols10cents:grammar-oops, r=alexcrichton<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(plugin, rustc_private, str_char, collections)]

extern crate syntax;
extern crate rustc;

#[macro_use]
extern crate log;

use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufRead, Read};
use std::path::Path;

use syntax::parse;
use syntax::parse::lexer;
use rustc::session::{self, config};

use syntax::ast;
use syntax::ast::Name;
use syntax::codemap;
use syntax::codemap::Pos;
use syntax::parse::token;
use syntax::parse::lexer::TokenAndSpan;

fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
    fn id() -> token::Token {
        token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
    }

    let mut res = HashMap::new();

    res.insert("-1".to_string(), token::Eof);

    for line in file.split('\n') {
        let eq = match line.trim().rfind('=') {
            Some(val) => val,
            None => continue
        };

        let val = &line[..eq];
        let num = &line[eq + 1..];

        let tok = match val {
            "SHR"               => token::BinOp(token::Shr),
            "DOLLAR"            => token::Dollar,
            "LT"                => token::Lt,
            "STAR"              => token::BinOp(token::Star),
            "FLOAT_SUFFIX"      => id(),
            "INT_SUFFIX"        => id(),
            "SHL"               => token::BinOp(token::Shl),
            "LBRACE"            => token::OpenDelim(token::Brace),
            "RARROW"            => token::RArrow,
            "LIT_STR"           => token::Literal(token::Str_(Name(0)), None),
            "DOTDOT"            => token::DotDot,
            "MOD_SEP"           => token::ModSep,
            "DOTDOTDOT"         => token::DotDotDot,
            "NOT"               => token::Not,
            "AND"               => token::BinOp(token::And),
            "LPAREN"            => token::OpenDelim(token::Paren),
            "ANDAND"            => token::AndAnd,
            "AT"                => token::At,
            "LBRACKET"          => token::OpenDelim(token::Bracket),
            "LIT_STR_RAW"       => token::Literal(token::StrRaw(Name(0), 0), None),
            "RPAREN"            => token::CloseDelim(token::Paren),
            "SLASH"             => token::BinOp(token::Slash),
            "COMMA"             => token::Comma,
            "LIFETIME"          => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
            "CARET"             => token::BinOp(token::Caret),
            "TILDE"             => token::Tilde,
            "IDENT"             => id(),
            "PLUS"              => token::BinOp(token::Plus),
            "LIT_CHAR"          => token::Literal(token::Char(Name(0)), None),
            "LIT_BYTE"          => token::Literal(token::Byte(Name(0)), None),
            "EQ"                => token::Eq,
            "RBRACKET"          => token::CloseDelim(token::Bracket),
            "COMMENT"           => token::Comment,
            "DOC_COMMENT"       => token::DocComment(Name(0)),
            "DOT"               => token::Dot,
            "EQEQ"              => token::EqEq,
            "NE"                => token::Ne,
            "GE"                => token::Ge,
            "PERCENT"           => token::BinOp(token::Percent),
            "RBRACE"            => token::CloseDelim(token::Brace),
            "BINOP"             => token::BinOp(token::Plus),
            "POUND"             => token::Pound,
            "OROR"              => token::OrOr,
            "LIT_INTEGER"       => token::Literal(token::Integer(Name(0)), None),
            "BINOPEQ"           => token::BinOpEq(token::Plus),
            "LIT_FLOAT"         => token::Literal(token::Float(Name(0)), None),
            "WHITESPACE"        => token::Whitespace,
            "UNDERSCORE"        => token::Underscore,
            "MINUS"             => token::BinOp(token::Minus),
            "SEMI"              => token::Semi,
            "COLON"             => token::Colon,
            "FAT_ARROW"         => token::FatArrow,
            "OR"                => token::BinOp(token::Or),
            "GT"                => token::Gt,
            "LE"                => token::Le,
            "LIT_BINARY"        => token::Literal(token::Binary(Name(0)), None),
            "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0), None),
            "QUESTION"          => token::Question,
            "SHEBANG"           => token::Shebang(Name(0)),
            _                   => continue,
        };

        res.insert(num.to_string(), tok);
    }

    debug!("Token map: {:?}", res);
    res
}

fn str_to_binop(s: &str) -> token::BinOpToken {
    match s {
        "+"     => token::Plus,
        "/"     => token::Slash,
        "-"     => token::Minus,
        "*"     => token::Star,
        "%"     => token::Percent,
        "^"     => token::Caret,
        "&"     => token::And,
        "|"     => token::Or,
        "<<"    => token::Shl,
        ">>"    => token::Shr,
        _       => panic!("Bad binop str `{}`", s),
    }
}

/// Assuming a string/binary literal, strip out the leading/trailing
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'r' {
        if lit.char_at(1) == 'b' {
            lit = &lit[2..]
        } else {
            lit = &lit[1..];
        }
    } else if lit.char_at(0) == 'b' {
        lit = &lit[1..];
    }

    let leading_hashes = count(lit);

    // +1/-1 to adjust for single quotes
    parse::token::intern(&lit[leading_hashes + 1..lit.len() - leading_hashes - 1])
}

/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'b' {
        lit = &lit[1..];
    }

    parse::token::intern(&lit[1..lit.len() - 1])
}

fn count(lit: &str) -> usize {
    lit.chars().take_while(|c| *c == '#').count()
}

fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize],
                     has_bom: bool)
                     -> TokenAndSpan {
    // old regex:
    // \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
    let start = s.find("[@").unwrap();
    let comma = start + s[start..].find(",").unwrap();
    let colon = comma + s[comma..].find(":").unwrap();
    let content_start = colon + s[colon..].find("='").unwrap();
    // Use rfind instead of find, because we don't want to stop at the content
    let content_end = content_start + s[content_start..].rfind("',<").unwrap();
    let toknum_end = content_end + s[content_end..].find(">,").unwrap();

    let start = &s[comma + 1 .. colon];
    let end = &s[colon + 1 .. content_start];
    let content = &s[content_start + 2 .. content_end];
    let toknum = &s[content_end + 3 .. toknum_end];

    let not_found = format!("didn't find token {:?} in the map", toknum);
    let proto_tok = tokens.get(toknum).expect(&not_found[..]);

    let nm = parse::token::intern(content);

    debug!("What we got: content (`{}`), proto: {:?}", content, proto_tok);

    let real_tok = match *proto_tok {
        token::BinOp(..)           => token::BinOp(str_to_binop(content)),
        token::BinOpEq(..)         => token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
        token::Literal(token::Str_(..), n)      => token::Literal(token::Str_(fix(content)), n),
        token::Literal(token::StrRaw(..), n)    => token::Literal(token::StrRaw(fix(content),
                                                                             count(content)), n),
        token::Literal(token::Char(..), n)      => token::Literal(token::Char(fixchar(content)), n),
        token::Literal(token::Byte(..), n)      => token::Literal(token::Byte(fixchar(content)), n),
        token::DocComment(..)      => token::DocComment(nm),
        token::Literal(token::Integer(..), n)   => token::Literal(token::Integer(nm), n),
        token::Literal(token::Float(..), n)     => token::Literal(token::Float(nm), n),
        token::Literal(token::Binary(..), n)    => token::Literal(token::Binary(nm), n),
        token::Literal(token::BinaryRaw(..), n) => token::Literal(token::BinaryRaw(fix(content),
                                                                                count(content)), n),
        token::Ident(..)           => token::Ident(ast::Ident { name: nm, ctxt: 0 },
                                                   token::ModName),
        token::Lifetime(..)        => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
        ref t => t.clone()
    };

    let start_offset = if real_tok == token::Eof {
        1
    } else {
        0
    };

    let offset = if has_bom { 1 } else { 0 };

    let mut lo = start.parse::<u32>().unwrap() - start_offset - offset;
    let mut hi = end.parse::<u32>().unwrap() + 1 - offset;

    // Adjust the span: For each surrogate pair already encountered, subtract one position.
    lo -= surrogate_pairs_pos.binary_search(&(lo as usize)).unwrap_or_else(|x| x) as u32;
    hi -= surrogate_pairs_pos.binary_search(&(hi as usize)).unwrap_or_else(|x| x) as u32;

    let sp = codemap::Span {
        lo: codemap::BytePos(lo),
        hi: codemap::BytePos(hi),
        expn_id: codemap::NO_EXPANSION
    };

    TokenAndSpan {
        tok: real_tok,
        sp: sp
    }
}

fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
    match a {
        &token::Ident(id, _) => match b {
                &token::Ident(id2, _) => id == id2,
                _ => false
        },
        _ => a == b
    }
}

fn span_cmp(antlr_sp: codemap::Span, rust_sp: codemap::Span, cm: &codemap::CodeMap) -> bool {
    antlr_sp.expn_id == rust_sp.expn_id &&
        antlr_sp.lo.to_usize() == cm.bytepos_to_file_charpos(rust_sp.lo).to_usize() &&
        antlr_sp.hi.to_usize() == cm.bytepos_to_file_charpos(rust_sp.hi).to_usize()
}

fn main() {
    fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
        use syntax::parse::lexer::Reader;
        r.next_token()
    }

    let mut args = env::args().skip(1);
    let filename = args.next().unwrap();
    if filename.find("parse-fail").is_some() {
        return;
    }

    // Rust's lexer
    let mut code = String::new();
    File::open(&Path::new(&filename)).unwrap().read_to_string(&mut code).unwrap();

    let surrogate_pairs_pos: Vec<usize> = code.chars().enumerate()
                                                     .filter(|&(_, c)| c as usize > 0xFFFF)
                                                     .map(|(n, _)| n)
                                                     .enumerate()
                                                     .map(|(x, n)| x + n)
                                                     .collect();

    let has_bom = code.starts_with("\u{feff}");

    debug!("Pairs: {:?}", surrogate_pairs_pos);

    let options = config::basic_options();
    let session = session::build_session(options, None,
                                         syntax::diagnostics::registry::Registry::new(&[]));
    let filemap = session.parse_sess.codemap().new_filemap(String::from_str("<n/a>"), code);
    let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
    let cm = session.codemap();

    // ANTLR
    let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();
    let mut token_list = String::new();
    token_file.read_to_string(&mut token_list).unwrap();
    let token_map = parse_token_list(&token_list[..]);

    let stdin = std::io::stdin();
    let lock = stdin.lock();
    let lines = lock.lines();
    let antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
                                                       &token_map,
                                                       &surrogate_pairs_pos[..],
                                                       has_bom));

    for antlr_tok in antlr_tokens {
        let rustc_tok = next(&mut lexer);
        if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
            continue
        }

        assert!(span_cmp(antlr_tok.sp, rustc_tok.sp, cm), "{:?} and {:?} have different spans",
                rustc_tok,
                antlr_tok);

        macro_rules! matches {
            ( $($x:pat),+ ) => (
                match rustc_tok.tok {
                    $($x => match antlr_tok.tok {
                        $x => {
                            if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
                                // FIXME #15677: needs more robust escaping in
                                // antlr
                                warn!("Different names for {:?} and {:?}", rustc_tok, antlr_tok);
                            }
                        }
                        _ => panic!("{:?} is not {:?}", antlr_tok, rustc_tok)
                    },)*
                    ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", antlr_tok, rustc_tok)
                }
            )
        }

        matches!(
            token::Literal(token::Byte(..), _),
            token::Literal(token::Char(..), _),
            token::Literal(token::Integer(..), _),
            token::Literal(token::Float(..), _),
            token::Literal(token::Str_(..), _),
            token::Literal(token::StrRaw(..), _),
            token::Literal(token::Binary(..), _),
            token::Literal(token::BinaryRaw(..), _),
            token::Ident(..),
            token::Lifetime(..),
            token::Interpolated(..),
            token::DocComment(..),
            token::Shebang(..)
        );
    }
}
<|endoftext|>