<commit_before>use std::cmp::{self, Ordering};
use std::collections::hash_map::{HashMap, Values, IterMut};
use std::fmt::{self, Formatter};
use std::hash;
use std::mem;
use std::path::Path;
use std::sync::Arc;
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};

use url::Url;

use core::{Package, PackageId, Registry};
use sources::{PathSource, GitSource, RegistrySource};
use sources::git;
use util::{human, Config, CargoResult, ToUrl};

/// A Source finds and downloads remote packages based on names and
/// versions.
pub trait Source: Registry {
    /// The update method performs any network operations required to
    /// get the entire list of all names, versions and dependencies of
    /// packages managed by the Source.
    fn update(&mut self) -> CargoResult<()>;

    /// The download method fetches the full package for each name and
    /// version specified.
    fn download(&mut self, package: &PackageId) -> CargoResult<Package>;

    /// Generates a unique string which represents the fingerprint of the
    /// current state of the source.
    ///
    /// This fingerprint is used to determine the "fresheness" of the source
    /// later on. It must be guaranteed that the fingerprint of a source is
    /// constant if and only if the output product will remain constant.
    ///
    /// The `pkg` argument is the package which this fingerprint should only be
    /// interested in for when this source may contain multiple packages.
    fn fingerprint(&self, pkg: &Package) -> CargoResult<String>;
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
enum Kind {
    /// Kind::Git(<git reference>) represents a git repository
    Git(GitReference),
    /// represents a local path
    Path,
    /// represents the central registry
    Registry,
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum GitReference {
    Tag(String),
    Branch(String),
    Rev(String),
}

/// Unique identifier for a source of packages.
#[derive(Clone, Eq, Debug)]
pub struct SourceId {
    inner: Arc<SourceIdInner>,
}

#[derive(Eq, Clone, Debug)]
struct SourceIdInner {
    url: Url,
    canonical_url: Url,
    kind: Kind,
    // e.g. the exact git revision of the specified branch for a Git Source
    precise: Option<String>
}

impl SourceId {
    fn new(kind: Kind, url: Url) -> SourceId {
        SourceId {
            inner: Arc::new(SourceIdInner {
                kind: kind,
                canonical_url: git::canonicalize_url(&url),
                url: url,
                precise: None,
            }),
        }
    }

    /// Parses a source URL and returns the corresponding ID.
    ///
    /// ## Example
    ///
    /// ```
    /// use cargo::core::SourceId;
    /// SourceId::from_url("git+https://github.com/alexcrichton/\
    ///                     libssh2-static-sys#80e71a3021618eb05\
    ///                     656c58fb7c5ef5f12bc747f".to_string());
    /// ```
    pub fn from_url(string: String) -> SourceId {
        let mut parts = string.splitn(2, '+');
        let kind = parts.next().unwrap();
        let url = parts.next().unwrap();

        match kind {
            "git" => {
                let mut url = url.to_url().unwrap();
                let mut reference = GitReference::Branch("master".to_string());
                let pairs = url.query_pairs().unwrap_or(Vec::new());
                for &(ref k, ref v) in pairs.iter() {
                    match &k[..] {
                        // map older 'ref' to branch
                        "branch" |
                        "ref" => reference = GitReference::Branch(v.clone()),

                        "rev" => reference = GitReference::Rev(v.clone()),
                        "tag" => reference = GitReference::Tag(v.clone()),
                        _ => {}
                    }
                }
                url.query = None;
                let precise = mem::replace(&mut url.fragment, None);
                SourceId::for_git(&url, reference)
                         .with_precise(precise)
            },
            "registry" => {
                let url = url.to_url().unwrap();
                SourceId::new(Kind::Registry, url)
                         .with_precise(Some("locked".to_string()))
            }
            "path" => {
                let url = url.to_url().unwrap();
                SourceId::new(Kind::Path, url)
            }
            _ => panic!("Unsupported serialized SourceId")
        }
    }

    pub fn to_url(&self) -> String {
        match *self.inner {
            SourceIdInner { kind: Kind::Path, ref url, .. } => {
                format!("path+{}", url)
            }
            SourceIdInner {
                kind: Kind::Git(ref reference), ref url, ref precise, ..
            } => {
                let ref_str = url_ref(reference);

                let precise_str = if precise.is_some() {
                    format!("#{}", precise.as_ref().unwrap())
                } else {
                    "".to_string()
                };

                format!("git+{}{}{}", url, ref_str, precise_str)
            },
            SourceIdInner { kind: Kind::Registry, ref url, .. } => {
                format!("registry+{}", url)
            }
        }
    }

    // Pass absolute path
    pub fn for_path(path: &Path) -> CargoResult<SourceId> {
        let url = try!(path.to_url().map_err(human));
        Ok(SourceId::new(Kind::Path, url))
    }

    pub fn for_git(url: &Url, reference: GitReference) -> SourceId {
        SourceId::new(Kind::Git(reference), url.clone())
    }

    pub fn for_registry(url: &Url) -> SourceId {
        SourceId::new(Kind::Registry, url.clone())
    }

    /// Returns the `SourceId` corresponding to the main repository.
    ///
    /// This is the main cargo registry by default, but it can be overridden in
    /// a `.cargo/config`.
    pub fn for_central(config: &Config) -> CargoResult<SourceId> {
        Ok(SourceId::for_registry(&try!(RegistrySource::url(config))))
    }

    pub fn url(&self) -> &Url { &self.inner.url }
    pub fn is_path(&self) -> bool { self.inner.kind == Kind::Path }
    pub fn is_registry(&self) -> bool { self.inner.kind == Kind::Registry }

    pub fn is_git(&self) -> bool {
        match self.inner.kind {
            Kind::Git(_) => true,
            _ => false
        }
    }

    /// Creates an implementation of `Source` corresponding to this ID.
    pub fn load<'a>(&self, config: &'a Config) -> Box<Source+'a> {
        trace!("loading SourceId; {}", self);
        match self.inner.kind {
            Kind::Git(..) => Box::new(GitSource::new(self, config)),
            Kind::Path => {
                let path = match self.inner.url.to_file_path() {
                    Ok(p) => p,
                    Err(()) => panic!("path sources cannot be remote"),
                };
                Box::new(PathSource::new(&path, self, config))
            }
            Kind::Registry => Box::new(RegistrySource::new(self, config)),
        }
    }

    pub fn precise(&self) -> Option<&str> {
        self.inner.precise.as_ref().map(|s| &s[..])
    }

    pub fn git_reference(&self) -> Option<&GitReference> {
        match self.inner.kind {
            Kind::Git(ref s) => Some(s),
            _ => None,
        }
    }

    pub fn with_precise(&self, v: Option<String>) -> SourceId {
        SourceId {
            inner: Arc::new(SourceIdInner {
                precise: v,
                .. (*self.inner).clone()
            }),
        }
    }

    pub fn is_default_registry(&self) -> bool {
        match self.inner.kind {
            Kind::Registry => {}
            _ => return false,
        }
        self.inner.url.to_string() == RegistrySource::default_url()
    }
}

impl PartialEq for SourceId {
    fn eq(&self, other: &SourceId) -> bool {
        (*self.inner).eq(&*other.inner)
    }
}

impl PartialOrd for SourceId {
    fn partial_cmp(&self, other: &SourceId) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for SourceId {
    fn cmp(&self, other: &SourceId) -> Ordering {
        self.inner.cmp(&other.inner)
    }
}

impl Encodable for SourceId {
    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
        if self.is_path() {
            s.emit_option_none()
        } else {
           self.to_url().encode(s)
        }
    }
}

impl Decodable for SourceId {
    fn decode<D: Decoder>(d: &mut D) -> Result<SourceId, D::Error> {
        let string: String = Decodable::decode(d).ok().expect("Invalid encoded SourceId");
        Ok(SourceId::from_url(string))
    }
}

impl fmt::Display for SourceId {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self.inner {
            SourceIdInner { kind: Kind::Path, ref url, .. } => {
                fmt::Display::fmt(url, f)
            }
            SourceIdInner { kind: Kind::Git(ref reference), ref url,
                            ref precise, .. } => {
                try!(write!(f, "{}{}", url, url_ref(reference)));

                if let Some(ref s) = *precise {
                    let len = cmp::min(s.len(), 8);
                    try!(write!(f, "#{}", &s[..len]));
                }
                Ok(())
            }
            SourceIdInner { kind: Kind::Registry, ref url, .. } => {
                write!(f, "registry {}", url)
            }
        }
    }
}

// This custom implementation handles situations such as when two git sources
// point at *almost* the same URL, but not quite, even when they actually point
// to the same repository.
impl PartialEq for SourceIdInner {
    fn eq(&self, other: &SourceIdInner) -> bool {
        if self.kind != other.kind { return false }
        if self.url == other.url { return true }

        match (&self.kind, &other.kind) {
            (&Kind::Git(ref ref1), &Kind::Git(ref ref2)) => {
                ref1 == ref2 && self.canonical_url == other.canonical_url
            }
            _ => false,
        }
    }
}

impl PartialOrd for SourceIdInner {
    fn partial_cmp(&self, other: &SourceIdInner) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for SourceIdInner {
    fn cmp(&self, other: &SourceIdInner) -> Ordering {
        match self.kind.cmp(&other.kind) {
            Ordering::Equal => {}
            ord => return ord,
        }
        match self.url.cmp(&other.url) {
            Ordering::Equal => {}
            ord => return ord,
        }
        match (&self.kind, &other.kind) {
            (&Kind::Git(ref ref1), &Kind::Git(ref ref2)) => {
                (ref1, &self.canonical_url).cmp(&(ref2, &other.canonical_url))
            }
            _ => self.kind.cmp(&other.kind)
        }
    }
}

impl hash::Hash for SourceId {
    fn hash<S: hash::Hasher>(&self, into: &mut S) {
        self.inner.kind.hash(into);
        match *self.inner {
            SourceIdInner { kind: Kind::Git(..), ref canonical_url, .. } => {
                canonical_url.hash(into)
            }
            _ => self.inner.url.hash(into),
        }
    }
}

fn url_ref(r: &GitReference) -> String {
    match r.to_ref_string() {
        None => "".to_string(),
        Some(s) => format!("?{}", s),
    }
}

impl GitReference {
    pub fn to_ref_string(&self) -> Option<String> {
        match *self {
            GitReference::Branch(ref s) => {
                if *s == "master" {
                    None
                } else {
                    Some(format!("branch={}", s))
                }
            }
            GitReference::Tag(ref s) => Some(format!("tag={}", s)),
            GitReference::Rev(ref s) => Some(format!("rev={}", s)),
        }
    }
}

pub struct SourceMap<'src> {
    map: HashMap<SourceId, Box<Source+'src>>
}

pub type Sources<'a, 'src> = Values<'a, SourceId, Box<Source+'src>>;

pub struct SourcesMut<'a, 'src: 'a> {
    inner: IterMut<'a, SourceId, Box<Source + 'src>>,
}

impl<'src> SourceMap<'src> {
    pub fn new() -> SourceMap<'src> {
        SourceMap {
            map: HashMap::new()
        }
    }

    pub fn contains(&self, id: &SourceId) -> bool {
        self.map.contains_key(id)
    }

    pub fn get(&self, id: &SourceId) -> Option<&(Source+'src)> {
        let source = self.map.get(id);

        source.map(|s| {
            let s: &(Source+'src) = &**s;
            s
        })
    }

    pub fn get_mut(&mut self, id: &SourceId) -> Option<&mut (Source+'src)> {
        self.map.get_mut(id).map(|s| {
            let s: &mut (Source+'src) = &mut **s;
            s
        })
    }

    pub fn get_by_package_id(&self, pkg_id: &PackageId) -> Option<&(Source+'src)> {
        self.get(pkg_id.source_id())
    }

    pub fn insert(&mut self, id: &SourceId, source: Box<Source+'src>) {
        self.map.insert(id.clone(), source);
    }

    pub fn is_empty(&self) -> bool {
        self.map.is_empty()
    }

    pub fn len(&self) -> usize {
        self.map.len()
    }

    pub fn sources<'a>(&'a self) -> Sources<'a, 'src> {
        self.map.values()
    }

    pub fn sources_mut<'a>(&'a mut self) -> SourcesMut<'a, 'src> {
        SourcesMut { inner: self.map.iter_mut() }
    }
}

impl<'a, 'src> Iterator for SourcesMut<'a, 'src> {
    type Item = (&'a SourceId, &'a mut (Source + 'src));
    fn next(&mut self) -> Option<(&'a SourceId, &'a mut (Source + 'src))> {
        self.inner.next().map(|(a, b)| (a, &mut **b))
    }
}

#[cfg(test)]
mod tests {
    use super::{SourceId, Kind, GitReference};
    use util::ToUrl;

    #[test]
    fn github_sources_equal() {
        let loc = "https://github.com/foo/bar".to_url().unwrap();
        let master = Kind::Git(GitReference::Branch("master".to_string()));
        let s1 = SourceId::new(master.clone(), loc);

        let loc = "git://github.com/foo/bar".to_url().unwrap();
        let s2 = SourceId::new(master, loc.clone());

        assert_eq!(s1, s2);

        let foo = Kind::Git(GitReference::Branch("foo".to_string()));
        let s3 = SourceId::new(foo, loc);
        assert!(s1 != s3);
    }
}
<commit_msg>run rustfmt on core/source.rs<commit_after>use std::cmp::{self, Ordering};
use std::collections::hash_map::{HashMap, Values, IterMut};
use std::fmt::{self, Formatter};
use std::hash;
use std::mem;
use std::path::Path;
use std::sync::Arc;
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};

use url::Url;

use core::{Package, PackageId, Registry};
use sources::{PathSource, GitSource, RegistrySource};
use sources::git;
use util::{human, Config, CargoResult, ToUrl};

/// A Source finds and downloads remote packages based on names and
/// versions.
pub trait Source: Registry {
    /// The update method performs any network operations required to
    /// get the entire list of all names, versions and dependencies of
    /// packages managed by the Source.
    fn update(&mut self) -> CargoResult<()>;

    /// The download method fetches the full package for each name and
    /// version specified.
    fn download(&mut self, package: &PackageId) -> CargoResult<Package>;

    /// Generates a unique string which represents the fingerprint of the
    /// current state of the source.
    ///
    /// This fingerprint is used to determine the "fresheness" of the source
    /// later on. It must be guaranteed that the fingerprint of a source is
    /// constant if and only if the output product will remain constant.
    ///
    /// The `pkg` argument is the package which this fingerprint should only be
    /// interested in for when this source may contain multiple packages.
    fn fingerprint(&self, pkg: &Package) -> CargoResult<String>;
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
enum Kind {
    /// Kind::Git(<git reference>) represents a git repository
    Git(GitReference),
    /// represents a local path
    Path,
    /// represents the central registry
    Registry,
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum GitReference {
    Tag(String),
    Branch(String),
    Rev(String),
}

/// Unique identifier for a source of packages.
#[derive(Clone, Eq, Debug)]
pub struct SourceId {
    inner: Arc<SourceIdInner>,
}

#[derive(Eq, Clone, Debug)]
struct SourceIdInner {
    url: Url,
    canonical_url: Url,
    kind: Kind,
    // e.g. the exact git revision of the specified branch for a Git Source
    precise: Option<String>,
}

impl SourceId {
    fn new(kind: Kind, url: Url) -> SourceId {
        SourceId {
            inner: Arc::new(SourceIdInner {
                kind: kind,
                canonical_url: git::canonicalize_url(&url),
                url: url,
                precise: None,
            }),
        }
    }

    /// Parses a source URL and returns the corresponding ID.
    ///
    /// ## Example
    ///
    /// ```
    /// use cargo::core::SourceId;
    /// SourceId::from_url("git+https://github.com/alexcrichton/\
    ///                     libssh2-static-sys#80e71a3021618eb05\
    ///                     656c58fb7c5ef5f12bc747f".to_string());
    /// ```
    pub fn from_url(string: String) -> SourceId {
        let mut parts = string.splitn(2, '+');
        let kind = parts.next().unwrap();
        let url = parts.next().unwrap();

        match kind {
            "git" => {
                let mut url = url.to_url().unwrap();
                let mut reference = GitReference::Branch("master".to_string());
                let pairs = url.query_pairs().unwrap_or(Vec::new());
                for &(ref k, ref v) in pairs.iter() {
                    match &k[..] {
                        // map older 'ref' to branch
                        "branch" |
                        "ref" => reference = GitReference::Branch(v.clone()),

                        "rev" => reference = GitReference::Rev(v.clone()),
                        "tag" => reference = GitReference::Tag(v.clone()),
                        _ => {}
                    }
                }
                url.query = None;
                let precise = mem::replace(&mut url.fragment, None);
                SourceId::for_git(&url, reference).with_precise(precise)
            }
            "registry" => {
                let url = url.to_url().unwrap();
                SourceId::new(Kind::Registry, url)
                    .with_precise(Some("locked".to_string()))
            }
            "path" => {
                let url = url.to_url().unwrap();
                SourceId::new(Kind::Path, url)
            }
            _ => panic!("Unsupported serialized SourceId"),
        }
    }

    pub fn to_url(&self) -> String {
        match *self.inner {
            SourceIdInner { kind: Kind::Path, ref url, .. } => {
                format!("path+{}", url)
            }
            SourceIdInner {
                kind: Kind::Git(ref reference), ref url, ref precise, ..
            } => {
                let ref_str = url_ref(reference);

                let precise_str = if precise.is_some() {
                    format!("#{}", precise.as_ref().unwrap())
                } else {
                    "".to_string()
                };

                format!("git+{}{}{}", url, ref_str, precise_str)
            }
            SourceIdInner { kind: Kind::Registry, ref url, .. } => {
                format!("registry+{}", url)
            }
        }
    }

    // Pass absolute path
    pub fn for_path(path: &Path) -> CargoResult<SourceId> {
        let url = try!(path.to_url().map_err(human));
        Ok(SourceId::new(Kind::Path, url))
    }

    pub fn for_git(url: &Url, reference: GitReference) -> SourceId {
        SourceId::new(Kind::Git(reference), url.clone())
    }

    pub fn for_registry(url: &Url) -> SourceId {
        SourceId::new(Kind::Registry, url.clone())
    }

    /// Returns the `SourceId` corresponding to the main repository.
    ///
    /// This is the main cargo registry by default, but it can be overridden in
    /// a `.cargo/config`.
    pub fn for_central(config: &Config) -> CargoResult<SourceId> {
        Ok(SourceId::for_registry(&try!(RegistrySource::url(config))))
    }

    pub fn url(&self) -> &Url {
        &self.inner.url
    }
    pub fn is_path(&self) -> bool {
        self.inner.kind == Kind::Path
    }
    pub fn is_registry(&self) -> bool {
        self.inner.kind == Kind::Registry
    }

    pub fn is_git(&self) -> bool {
        match self.inner.kind {
            Kind::Git(_) => true,
            _ => false,
        }
    }

    /// Creates an implementation of `Source` corresponding to this ID.
    pub fn load<'a>(&self, config: &'a Config) -> Box<Source + 'a> {
        trace!("loading SourceId; {}", self);
        match self.inner.kind {
            Kind::Git(..) => Box::new(GitSource::new(self, config)),
            Kind::Path => {
                let path = match self.inner.url.to_file_path() {
                    Ok(p) => p,
                    Err(()) => panic!("path sources cannot be remote"),
                };
                Box::new(PathSource::new(&path, self, config))
            }
            Kind::Registry => Box::new(RegistrySource::new(self, config)),
        }
    }

    pub fn precise(&self) -> Option<&str> {
        self.inner.precise.as_ref().map(|s| &s[..])
    }

    pub fn git_reference(&self) -> Option<&GitReference> {
        match self.inner.kind {
            Kind::Git(ref s) => Some(s),
            _ => None,
        }
    }

    pub fn with_precise(&self, v: Option<String>) -> SourceId {
        SourceId {
            inner: Arc::new(SourceIdInner {
                precise: v,
                ..(*self.inner).clone()
            })
        }
    }

    pub fn is_default_registry(&self) -> bool {
        match self.inner.kind {
            Kind::Registry => {}
            _ => return false,
        }
        self.inner.url.to_string() == RegistrySource::default_url()
    }
}

impl PartialEq for SourceId {
    fn eq(&self, other: &SourceId) -> bool {
        (*self.inner).eq(&*other.inner)
    }
}

impl PartialOrd for SourceId {
    fn partial_cmp(&self, other: &SourceId) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for SourceId {
    fn cmp(&self, other: &SourceId) -> Ordering {
        self.inner.cmp(&other.inner)
    }
}

impl Encodable for SourceId {
    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
        if self.is_path() {
            s.emit_option_none()
        } else {
            self.to_url().encode(s)
        }
    }
}

impl Decodable for SourceId {
    fn decode<D: Decoder>(d: &mut D) -> Result<SourceId, D::Error> {
        let string: String = Decodable::decode(d).ok().expect("Invalid encoded SourceId");
        Ok(SourceId::from_url(string))
    }
}

impl fmt::Display for SourceId {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self.inner {
            SourceIdInner { kind: Kind::Path, ref url, .. } => {
                fmt::Display::fmt(url, f)
            }
            SourceIdInner { kind: Kind::Git(ref reference), ref url,
                            ref precise, .. } => {
                try!(write!(f, "{}{}", url, url_ref(reference)));

                if let Some(ref s) = *precise {
                    let len = cmp::min(s.len(), 8);
                    try!(write!(f, "#{}", &s[..len]));
                }
                Ok(())
            }
            SourceIdInner { kind: Kind::Registry, ref url, .. } => {
                write!(f, "registry {}", url)
            }
        }
    }
}

// This custom implementation handles situations such as when two git sources
// point at *almost* the same URL, but not quite, even when they actually point
// to the same repository.
impl PartialEq for SourceIdInner {
    fn eq(&self, other: &SourceIdInner) -> bool {
        if self.kind != other.kind {
            return false;
        }
        if self.url == other.url {
            return true;
        }

        match (&self.kind, &other.kind) {
            (&Kind::Git(ref ref1), &Kind::Git(ref ref2)) => {
                ref1 == ref2 && self.canonical_url == other.canonical_url
            }
            _ => false,
        }
    }
}

impl PartialOrd for SourceIdInner {
    fn partial_cmp(&self, other: &SourceIdInner) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for SourceIdInner {
    fn cmp(&self, other: &SourceIdInner) -> Ordering {
        match self.kind.cmp(&other.kind) {
            Ordering::Equal => {}
            ord => return ord,
        }
        match self.url.cmp(&other.url) {
            Ordering::Equal => {}
            ord => return ord,
        }
        match (&self.kind, &other.kind) {
            (&Kind::Git(ref ref1), &Kind::Git(ref ref2)) => {
                (ref1, &self.canonical_url).cmp(&(ref2, &other.canonical_url))
            }
            _ => self.kind.cmp(&other.kind),
        }
    }
}

impl hash::Hash for SourceId {
    fn hash<S: hash::Hasher>(&self, into: &mut S) {
        self.inner.kind.hash(into);
        match *self.inner {
            SourceIdInner { kind: Kind::Git(..), ref canonical_url, .. } => {
                canonical_url.hash(into)
            }
            _ => self.inner.url.hash(into),
        }
    }
}

fn url_ref(r: &GitReference) -> String {
    match r.to_ref_string() {
        None => "".to_string(),
        Some(s) => format!("?{}", s),
    }
}

impl GitReference {
    pub fn to_ref_string(&self) -> Option<String> {
        match *self {
            GitReference::Branch(ref s) => {
                if *s == "master" {
                    None
                } else {
                    Some(format!("branch={}", s))
                }
            }
            GitReference::Tag(ref s) => Some(format!("tag={}", s)),
            GitReference::Rev(ref s) => Some(format!("rev={}", s)),
        }
    }
}

pub struct SourceMap<'src> {
    map: HashMap<SourceId, Box<Source + 'src>>,
}

pub type Sources<'a, 'src> = Values<'a, SourceId, Box<Source + 'src>>;

pub struct SourcesMut<'a, 'src: 'a> {
    inner: IterMut<'a, SourceId, Box<Source + 'src>>,
}

impl<'src> SourceMap<'src> {
    pub fn new() -> SourceMap<'src> {
        SourceMap { map: HashMap::new() }
    }

    pub fn contains(&self, id: &SourceId) -> bool {
        self.map.contains_key(id)
    }

    pub fn get(&self, id: &SourceId) -> Option<&(Source + 'src)> {
        let source = self.map.get(id);

        source.map(|s| {
            let s: &(Source + 'src) = &**s;
            s
        })
    }

    pub fn get_mut(&mut self, id: &SourceId) -> Option<&mut (Source + 'src)> {
        self.map.get_mut(id).map(|s| {
            let s: &mut (Source + 'src) = &mut **s;
            s
        })
    }

    pub fn get_by_package_id(&self, pkg_id: &PackageId) -> Option<&(Source + 'src)> {
        self.get(pkg_id.source_id())
    }

    pub fn insert(&mut self, id: &SourceId, source: Box<Source + 'src>) {
        self.map.insert(id.clone(), source);
    }

    pub fn is_empty(&self) -> bool {
        self.map.is_empty()
    }

    pub fn len(&self) -> usize {
        self.map.len()
    }

    pub fn sources<'a>(&'a self) -> Sources<'a, 'src> {
        self.map.values()
    }

    pub fn sources_mut<'a>(&'a mut self) -> SourcesMut<'a, 'src> {
        SourcesMut { inner: self.map.iter_mut() }
    }
}

impl<'a, 'src> Iterator for SourcesMut<'a, 'src> {
    type Item = (&'a SourceId, &'a mut (Source + 'src));
    fn next(&mut self) -> Option<(&'a SourceId, &'a mut (Source + 'src))> {
        self.inner.next().map(|(a, b)| (a, &mut **b))
    }
}

#[cfg(test)]
mod tests {
    use super::{SourceId, Kind, GitReference};
    use util::ToUrl;

    #[test]
    fn github_sources_equal() {
        let loc = "https://github.com/foo/bar".to_url().unwrap();
        let master = Kind::Git(GitReference::Branch("master".to_string()));
        let s1 = SourceId::new(master.clone(), loc);

        let loc = "git://github.com/foo/bar".to_url().unwrap();
        let s2 = SourceId::new(master, loc.clone());

        assert_eq!(s1, s2);

        let foo = Kind::Git(GitReference::Branch("foo".to_string()));
        let s3 = SourceId::new(foo, loc);
        assert!(s1 != s3);
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_greengrassv2::{Client, Error, Region, PKG_VERSION};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
struct Opt {
    /// The AWS Region.
    #[structopt(short, long)]
    region: Option<String>,

    /// Whether to display additional information.
    #[structopt(short, long)]
    verbose: bool,
}

// Lists your IoT cores.
// snippet-start:[iot.rust.list-core-devices]
async fn show_cores(client: &Client) -> Result<(), Error> {
    let resp = client.list_core_devices().send().await?;

    println!("cores:");

    for core in resp.core_devices().unwrap() {
        println!(
            "  Name:  {}",
            core.core_device_thing_name().unwrap_or_default()
        );
        println!(
            "  Status:  {:?}",
            core.status().unwrap()
        );
        println!(
            "  Last update:  {:?}",
            core.last_status_update_timestamp().unwrap()
        );
        println!();
    }

    println!();

    Ok(())
}
// snippet-end:[iot.rust.list-core-devices]

/// Lists the name, type, and ARN of your IoT cores in the Region.
///
/// # Arguments
///
/// * `[-r REGION]` - The Region in which the client is created.
///   If not supplied, uses the value of the **AWS_REGION** environment variable.
///   If the environment variable is not set, defaults to **us-west-2**.
/// * `[-v]` - Whether to display information.
#[tokio::main]
async fn main() -> Result<(), Error> {
    tracing_subscriber::fmt::init();
    let Opt { region, verbose } = Opt::from_args();

    let region_provider = RegionProviderChain::first_try(region.map(Region::new))
        .or_default_provider()
        .or_else(Region::new("us-west-2"));

    println!();
    if verbose {
        println!("IoT client version: {}", PKG_VERSION);
        println!(
            "Region:             {}",
            region_provider.region().await.unwrap().as_ref()
        );
        println!();
    }

    let shared_config = aws_config::from_env().region(region_provider).load().await;
    let client = Client::new(&shared_config);

    show_cores(&client).await
}
<commit_msg>cargo fmt<commit_after>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_greengrassv2::{Client, Error, Region, PKG_VERSION};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
struct Opt {
    /// The AWS Region.
    #[structopt(short, long)]
    region: Option<String>,

    /// Whether to display additional information.
    #[structopt(short, long)]
    verbose: bool,
}

// Lists your IoT cores.
// snippet-start:[iot.rust.list-core-devices]
async fn show_cores(client: &Client) -> Result<(), Error> {
    let resp = client.list_core_devices().send().await?;

    println!("cores:");

    for core in resp.core_devices().unwrap() {
        println!(
            "  Name:  {}",
            core.core_device_thing_name().unwrap_or_default()
        );
        println!("  Status:  {:?}", core.status().unwrap());
        println!(
            "  Last update:  {:?}",
            core.last_status_update_timestamp().unwrap()
        );
        println!();
    }

    println!();

    Ok(())
}
// snippet-end:[iot.rust.list-core-devices]

/// Lists the name, type, and ARN of your IoT cores in the Region.
///
/// # Arguments
///
/// * `[-r REGION]` - The Region in which the client is created.
///   If not supplied, uses the value of the **AWS_REGION** environment variable.
///   If the environment variable is not set, defaults to **us-west-2**.
/// * `[-v]` - Whether to display information.
#[tokio::main]
async fn main() -> Result<(), Error> {
    tracing_subscriber::fmt::init();
    let Opt { region, verbose } = Opt::from_args();

    let region_provider = RegionProviderChain::first_try(region.map(Region::new))
        .or_default_provider()
        .or_else(Region::new("us-west-2"));

    println!();
    if verbose {
        println!("IoT client version: {}", PKG_VERSION);
        println!(
            "Region:             {}",
            region_provider.region().await.unwrap().as_ref()
        );
        println!();
    }

    let shared_config = aws_config::from_env().region(region_provider).load().await;
    let client = Client::new(&shared_config);

    show_cores(&client).await
}
<|endoftext|>


<commit_before>use crate::{
    builtin_functions::BuiltinFunction,
    compiler::hir::{self, Expression},
    input::Input,
};

use super::{
    run::{Discover, DiscoverResult},
    value::{Environment, Value},
};

pub fn run_builtin_function(
    db: &dyn Discover,
    input: Input,
    builtin_function: BuiltinFunction,
    arguments: Vec<hir::Id>,
    environment: Environment,
) -> DiscoverResult {
    log::info!(
        "run_builtin_function: {:?} {}",
        builtin_function,
        arguments.len()
    );
    // Handle builtin functions that don't need to resolve the
    match builtin_function {
        BuiltinFunction::IfElse => return if_else(db, input, arguments, environment),
        _ => {}
    }

    let arguments = db.run_multiple_with_environment(input.to_owned(), arguments, environment)?;
    let arguments = match arguments {
        Ok(arguments) => arguments,
        Err(error) => return Some(Err(error)),
    };
    match builtin_function {
        BuiltinFunction::Add => add(arguments),
        BuiltinFunction::Equals => equals(arguments),
        BuiltinFunction::GetArgumentCount => get_argument_count(db, input, arguments),
        BuiltinFunction::Panic => panic(arguments),
        BuiltinFunction::Print => print(arguments),
        BuiltinFunction::TypeOf => type_of(arguments),
        _ => panic!("Unhandled builtin function: {:?}", builtin_function),
    }
}

macro_rules! destructure {
    ($arguments:expr, $enum:pat, $body:block) => {{
        if let $enum = &$arguments[..] {
            $body
        } else {
            Some(Err(Value::Text(format!("Invalid arguments").to_owned())))
        }
    }};
}

fn add(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [Value::Int(a), Value::Int(b)], {
        Some(Ok(Value::Int(a + b)))
    })
}

fn equals(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [a, b], { Some(Ok((a == b).into())) })
}

fn get_argument_count(db: &dyn Discover, input: Input, arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [Value::Lambda(function)], {
        // TODO: support parameter counts > 2^64 on 128-bit systems and better
        let expression = match db.find_expression(input, function.id.to_owned())? {
            Expression::Lambda(lambda) => lambda,
            _ => return None,
        };
        Some(Ok((expression.parameters.len() as u64).into()))
    })
}

fn if_else(
    db: &dyn Discover,
    input: Input,
    arguments: Vec<hir::Id>,
    environment: Environment,
) -> DiscoverResult {
    log::error!("{:?}", arguments);
    if let [condition, then, else_] = &arguments[..] {
        let body_id = match db.run_with_environment(
            input.clone(),
            condition.to_owned(),
            environment.to_owned(),
        )? {
            Ok(value) if value == Value::bool_true() => then,
            Ok(value) if value == Value::bool_false() => else_,
            Ok(_) => return None,
            Err(error) => return Some(Err(error)),
        };

        db.run_call(input, body_id.to_owned(), vec![], environment)
    } else {
        Some(Err(Value::Text(
            format!(
                "Builtin if/else called with wrong number of arguments: {}, expected: {}",
                arguments.len(),
                3
            )
            .into(),
        )))
    }
}

fn panic(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], { Some(Err(value.clone())) })
}

fn print(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], {
        println!("{:?}", value);
        Some(Ok(Value::nothing()))
    })
}

fn type_of(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], {
        match value {
            Value::Int(_) => Some(Ok(Value::Symbol("Int".to_owned()))),
            Value::Text(_) => Some(Ok(Value::Symbol("Text".to_owned()))),
            Value::Symbol(_) => Some(Ok(Value::Symbol("Symbol".to_owned()))),
            Value::Lambda(_) => Some(Ok(Value::Symbol("Function".to_owned()))),
        }
    })
}
<commit_msg>docs(compiler): add missing word in comment<commit_after>use crate::{
    builtin_functions::BuiltinFunction,
    compiler::hir::{self, Expression},
    input::Input,
};

use super::{
    run::{Discover, DiscoverResult},
    value::{Environment, Value},
};

pub fn run_builtin_function(
    db: &dyn Discover,
    input: Input,
    builtin_function: BuiltinFunction,
    arguments: Vec<hir::Id>,
    environment: Environment,
) -> DiscoverResult {
    log::info!(
        "run_builtin_function: {:?} {}",
        builtin_function,
        arguments.len()
    );
    // Handle builtin functions that don't need to resolve the arguments.
    match builtin_function {
        BuiltinFunction::IfElse => return if_else(db, input, arguments, environment),
        _ => {}
    }

    let arguments = db.run_multiple_with_environment(input.to_owned(), arguments, environment)?;
    let arguments = match arguments {
        Ok(arguments) => arguments,
        Err(error) => return Some(Err(error)),
    };
    match builtin_function {
        BuiltinFunction::Add => add(arguments),
        BuiltinFunction::Equals => equals(arguments),
        BuiltinFunction::GetArgumentCount => get_argument_count(db, input, arguments),
        BuiltinFunction::Panic => panic(arguments),
        BuiltinFunction::Print => print(arguments),
        BuiltinFunction::TypeOf => type_of(arguments),
        _ => panic!("Unhandled builtin function: {:?}", builtin_function),
    }
}

macro_rules! destructure {
    ($arguments:expr, $enum:pat, $body:block) => {{
        if let $enum = &$arguments[..] {
            $body
        } else {
            Some(Err(Value::Text(format!("Invalid arguments").to_owned())))
        }
    }};
}

fn add(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [Value::Int(a), Value::Int(b)], {
        Some(Ok(Value::Int(a + b)))
    })
}

fn equals(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [a, b], { Some(Ok((a == b).into())) })
}

fn get_argument_count(db: &dyn Discover, input: Input, arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [Value::Lambda(function)], {
        // TODO: support parameter counts > 2^64 on 128-bit systems and better
        let expression = match db.find_expression(input, function.id.to_owned())? {
            Expression::Lambda(lambda) => lambda,
            _ => return None,
        };
        Some(Ok((expression.parameters.len() as u64).into()))
    })
}

fn if_else(
    db: &dyn Discover,
    input: Input,
    arguments: Vec<hir::Id>,
    environment: Environment,
) -> DiscoverResult {
    log::error!("{:?}", arguments);
    if let [condition, then, else_] = &arguments[..] {
        let body_id = match db.run_with_environment(
            input.clone(),
            condition.to_owned(),
            environment.to_owned(),
        )? {
            Ok(value) if value == Value::bool_true() => then,
            Ok(value) if value == Value::bool_false() => else_,
            Ok(_) => return None,
            Err(error) => return Some(Err(error)),
        };

        db.run_call(input, body_id.to_owned(), vec![], environment)
    } else {
        Some(Err(Value::Text(
            format!(
                "Builtin if/else called with wrong number of arguments: {}, expected: {}",
                arguments.len(),
                3
            )
            .into(),
        )))
    }
}

fn panic(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], { Some(Err(value.clone())) })
}

fn print(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], {
        println!("{:?}", value);
        Some(Ok(Value::nothing()))
    })
}

fn type_of(arguments: Vec<Value>) -> DiscoverResult {
    destructure!(arguments, [value], {
        match value {
            Value::Int(_) => Some(Ok(Value::Symbol("Int".to_owned()))),
            Value::Text(_) => Some(Ok(Value::Symbol("Text".to_owned()))),
            Value::Symbol(_) => Some(Ok(Value::Symbol("Symbol".to_owned()))),
            Value::Lambda(_) => Some(Ok(Value::Symbol("Function".to_owned()))),
        }
    })
}
<|endoftext|>


<commit_before>// This is a part of rust-encoding.
// Copyright (c) 2013-2014, Kang Seonghoon.
// See README.md and LICENSE.txt for details.

/*!

# Rust-encoding

Character encoding support for Rust.
It is based on [WHATWG Encoding Standard](http://encoding.spec.whatwg.org/),
and also provides an advanced interface for error detection and recovery.

## Simple Usage

To encode a string:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.encode("caf\xe9", EncodeStrict),
           Ok(vec!(99,97,102,233)));
~~~~

To encode a string with unrepresentable characters:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict, EncodeReplace, EncodeIgnore, EncodeNcrEscape};
use encoding::all::ISO_8859_2;

assert!(ISO_8859_2.encode("Acme\xa9", EncodeStrict).is_err());
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeReplace),
           Ok(vec!(65,99,109,101,63)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeIgnore),
           Ok(vec!(65,99,109,101)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeNcrEscape),
           Ok(vec!(65,99,109,101,38,35,49,54,57,59)));
~~~~

To decode a byte sequence:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.decode([99,97,102,233], DecodeStrict),
           Ok("caf\xe9".to_string()));
~~~~

To decode a byte sequence with invalid sequences:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict, DecodeReplace, DecodeIgnore};
use encoding::all::ISO_8859_6;

assert!(ISO_8859_6.decode([65,99,109,101,169], DecodeStrict).is_err());
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeReplace),
           Ok("Acme\ufffd".to_string()));
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeIgnore),
           Ok("Acme".to_string()));
~~~~

A practical example of custom encoder traps:

~~~~ {.rust}
use encoding::{Encoding, Encoder, ByteWriter, EncoderTrap, EncoderCall, DecodeStrict};
use encoding::all::ASCII;

// hexadecimal numeric character reference replacement
fn hex_ncr_escape(_encoder: &mut Encoder, input: &str, output: &mut ByteWriter) -> bool {
    let escapes: Vec<String> =
        input.chars().map(|ch| format!("&#x{:x};", ch as int)).collect();
    let escapes = escapes.concat();
    output.write_bytes(escapes.as_bytes());
    true
}
static HexNcrEscape: EncoderTrap = EncoderCall(hex_ncr_escape);

let orig = "Hello, 世界!".to_string();
let encoded = ASCII.encode(orig[], HexNcrEscape).unwrap();
assert_eq!(ASCII.decode(encoded[], DecodeStrict),
           Ok("Hello, 世界!".to_string()));
~~~~

Getting the encoding from the string label, as specified in WHATWG Encoding standard:

~~~~ {.rust}
use encoding::{Encoding, DecodeReplace};
use encoding::label::encoding_from_whatwg_label;
use encoding::all::WINDOWS_949;

let euckr = encoding_from_whatwg_label("euc-kr").unwrap();
assert_eq!(euckr.name(), "windows-949");
assert_eq!(euckr.whatwg_name(), Some("euc-kr")); // for the sake of compatibility
let broken = &[0xbf, 0xec, 0xbf, 0xcd, 0xff, 0xbe, 0xd3];
assert_eq!(euckr.decode(broken, DecodeReplace),
           Ok("\uc6b0\uc640\ufffd\uc559".to_string()));

// corresponding rust-encoding native API:
assert_eq!(WINDOWS_949.decode(broken, DecodeReplace),
           Ok("\uc6b0\uc640\ufffd\uc559".to_string()));
~~~~

## Detailed Usage

There are three main entry points to rust-encoding.

**`Encoding`** is a single character encoding.
It contains `encode` and `decode` methods for converting `String` to `Vec<u8>` and vice versa.
For the error handling, they receive **traps** (`EncoderCall` and `DecoderCall` respectively)
which replace any error with some string (e.g. `U+FFFD`) or sequence (e.g. `?`).
You can also use `EncodeStrict` and `DecodeStrict` traps to stop on an error.

There are two ways to get `Encoding`:

* `encoding::all` has static items for every supported encoding.
  You should use them when the encoding would not change or only handful of them are required.
  Combined with link-time optimization, any unused encoding would be discarded from the binary.
* `encoding::label` has functions to dynamically get an encoding from given string ("label").
  They will return a static reference to the encoding, which type is also known as `EncodingRef`.
  It is useful when a list of required encodings is not available in advance,
  but it will result in the larger binary and missed optimization opportunities.

**`Encoder`** is an experimental incremental encoder.
At each step of `raw_feed`, it receives a slice of string
and emits any encoded bytes to a generic `ByteWriter` (normally `Vec<u8>`).
It will stop at the first error if any, and would return a `CodecError` struct in that case.
The caller is responsible for calling `raw_finish` at the end of encoding process.

**`Decoder`** is an experimental incremental decoder.
At each step of `raw_feed`, it receives a slice of byte sequence
and emits any decoded characters to a generic `StringWriter` (normally `String`).
Otherwise it is identical to `Encoder`s.

One should prefer `Encoding::{encode,decode}` as a primary interface.
`Encoder` and `Decoder` is experimental and can change substantially.
See the additional documents on `encoding::types` module for more information on them.

## Supported Encodings

Rust-encoding covers all encodings specified by WHATWG Encoding Standard and some more:

* 7-bit strict ASCII (`ascii`)
* UTF-8 (`utf-8`)
* UTF-16 in little endian (`utf-16` or `utf-16le`) and big endian (`utf-16be`)
* All single byte encoding in WHATWG Encoding Standard:
    * IBM code page 866
    * ISO 8859-{2,3,4,5,6,7,8,10,13,14,15,16}
    * KOI8-R, KOI8-U
    * MacRoman (`macintosh`), Macintosh Cyrillic encoding (`x-mac-cyrillic`)
    * Windows code pages 874, 1250, 1251, 1252 (instead of ISO 8859-1), 1253,
      1254 (instead of ISO 8859-9), 1255, 1256, 1257, 1258
* All multi byte encodings in WHATWG Encoding Standard:
    * Windows code page 949 (`euc-kr`, since the strict EUC-KR is hardly used)
    * EUC-JP and Windows code page 932 (`shift_jis`,
      since it's the most widespread extension to Shift_JIS)
    * ISO-2022-JP with asymmetric JIS X 0212 support
    * GB 18030
    * HZ
    * Big5-2003 with HKSCS-2008 extensions
* ISO 8859-1 (distinct from Windows code page 1252)

Parenthesized names refer to the encoding's primary name assigned by WHATWG Encoding Standard.

Many legacy character encodings lack the proper specification,
and even those that have a specification are highly dependent of the actual implementation.
Consequently one should be careful when picking a desired character encoding.
The only standards reliable in this regard are WHATWG Encoding Standard and
[vendor-provided mappings from the Unicode consortium](http://www.unicode.org/Public/MAPPINGS/).
Whenever in doubt, look at the source code and specifications for detailed explanations.

*/

#![crate_name = "encoding"]
#![crate_type = "lib"]
#![comment = "Character encoding support for Rust"]
#![license = "MIT"]

#![feature(globs, macro_rules)]

#![allow(experimental)]

// XXX temporarily cope with the nightly, see #47
#![allow(unknown_features)]
#![feature(slicing_syntax)]

#[cfg(test)] extern crate test;

pub use self::types::{CodecError, ByteWriter, StringWriter,
                      Encoder, Decoder, EncodingRef, Encoding,
                      EncoderTrapFunc, DecoderTrapFunc, DecoderTrap,
                      DecodeStrict, DecodeReplace, DecodeIgnore,
                      EncoderTrap, EncodeStrict, EncodeReplace,
                      EncodeIgnore, EncodeNcrEscape, decode,
                      EncoderCall, DecoderCall}; // reexport

mod util;
#[cfg(test)] mod testutils;

pub mod types;

/// Indices used for character encoding implementation. Semi-internal.
#[unstable]
pub mod index {
    pub mod ibm866;
    pub mod iso_8859_2;
    pub mod iso_8859_3;
    pub mod iso_8859_4;
    pub mod iso_8859_5;
    pub mod iso_8859_6;
    pub mod iso_8859_7;
    pub mod iso_8859_8;
    pub mod iso_8859_10;
    pub mod iso_8859_13;
    pub mod iso_8859_14;
    pub mod iso_8859_15;
    pub mod iso_8859_16;
    pub mod koi8_r;
    pub mod koi8_u;
    pub mod macintosh;
    pub mod windows_874;
    pub mod windows_1250;
    pub mod windows_1251;
    pub mod windows_1252;
    pub mod windows_1253;
    pub mod windows_1254;
    pub mod windows_1255;
    pub mod windows_1256;
    pub mod windows_1257;
    pub mod windows_1258;
    pub mod x_mac_cyrillic;
    pub mod big5;
    pub mod euc_kr;
    pub mod gb18030;
    pub mod gb18030_ranges;
    pub mod jis0208;
    pub mod jis0212;
}

/// Codec implementations.
#[unstable]
pub mod codec {
    pub mod error;
    pub mod ascii;
    pub mod singlebyte;
    pub mod utf_8;
    pub mod utf_16;
    pub mod korean;
    pub mod japanese;
    pub mod simpchinese;
    pub mod tradchinese;
    pub mod whatwg;
}

pub mod all;
pub mod label;

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_decode() {
        fn test_one(input: &[u8], expected_result: &str, expected_encoding: &str) {
            let (result, used_encoding) = decode(
                input, DecodeStrict, all::ISO_8859_1 as EncodingRef);
            let result = result.unwrap();
            assert_eq!(used_encoding.name(), expected_encoding);
            assert_eq!(result[], expected_result);
        }

        test_one([0xEF, 0xBB, 0xBF, 0xC3, 0xA9], "é", "utf-8");
        test_one([0xC3, 0xA9], "Ã©", "iso-8859-1");

        test_one([0xFE, 0xFF, 0x00, 0xE9], "é", "utf-16be");
        test_one([0x00, 0xE9], "\x00é", "iso-8859-1");

        test_one([0xFF, 0xFE, 0xE9, 0x00], "é", "utf-16le");
        test_one([0xE9, 0x00], "é\x00", "iso-8859-1");
    }
}

<commit_msg>doctest needs #![feature] as well.<commit_after>// This is a part of rust-encoding.
// Copyright (c) 2013-2014, Kang Seonghoon.
// See README.md and LICENSE.txt for details.

/*!

# Rust-encoding

Character encoding support for Rust.
It is based on [WHATWG Encoding Standard](http://encoding.spec.whatwg.org/),
and also provides an advanced interface for error detection and recovery.

## Simple Usage

To encode a string:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.encode("caf\xe9", EncodeStrict),
           Ok(vec!(99,97,102,233)));
~~~~

To encode a string with unrepresentable characters:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict, EncodeReplace, EncodeIgnore, EncodeNcrEscape};
use encoding::all::ISO_8859_2;

assert!(ISO_8859_2.encode("Acme\xa9", EncodeStrict).is_err());
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeReplace),
           Ok(vec!(65,99,109,101,63)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeIgnore),
           Ok(vec!(65,99,109,101)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeNcrEscape),
           Ok(vec!(65,99,109,101,38,35,49,54,57,59)));
~~~~

To decode a byte sequence:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.decode([99,97,102,233], DecodeStrict),
           Ok("caf\xe9".to_string()));
~~~~

To decode a byte sequence with invalid sequences:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict, DecodeReplace, DecodeIgnore};
use encoding::all::ISO_8859_6;

assert!(ISO_8859_6.decode([65,99,109,101,169], DecodeStrict).is_err());
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeReplace),
           Ok("Acme\ufffd".to_string()));
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeIgnore),
           Ok("Acme".to_string()));
~~~~

A practical example of custom encoder traps:

~~~~ {.rust}
# #![feature(slicing_syntax)]
use encoding::{Encoding, Encoder, ByteWriter, EncoderTrap, EncoderCall, DecodeStrict};
use encoding::all::ASCII;

// hexadecimal numeric character reference replacement
fn hex_ncr_escape(_encoder: &mut Encoder, input: &str, output: &mut ByteWriter) -> bool {
    let escapes: Vec<String> =
        input.chars().map(|ch| format!("&#x{:x};", ch as int)).collect();
    let escapes = escapes.concat();
    output.write_bytes(escapes.as_bytes());
    true
}
static HexNcrEscape: EncoderTrap = EncoderCall(hex_ncr_escape);

let orig = "Hello, 世界!".to_string();
let encoded = ASCII.encode(orig[], HexNcrEscape).unwrap();
assert_eq!(ASCII.decode(encoded[], DecodeStrict),
           Ok("Hello, 世界!".to_string()));
~~~~

Getting the encoding from the string label, as specified in WHATWG Encoding standard:

~~~~ {.rust}
use encoding::{Encoding, DecodeReplace};
use encoding::label::encoding_from_whatwg_label;
use encoding::all::WINDOWS_949;

let euckr = encoding_from_whatwg_label("euc-kr").unwrap();
assert_eq!(euckr.name(), "windows-949");
assert_eq!(euckr.whatwg_name(), Some("euc-kr")); // for the sake of compatibility
let broken = &[0xbf, 0xec, 0xbf, 0xcd, 0xff, 0xbe, 0xd3];
assert_eq!(euckr.decode(broken, DecodeReplace),
           Ok("\uc6b0\uc640\ufffd\uc559".to_string()));

// corresponding rust-encoding native API:
assert_eq!(WINDOWS_949.decode(broken, DecodeReplace),
           Ok("\uc6b0\uc640\ufffd\uc559".to_string()));
~~~~

## Detailed Usage

There are three main entry points to rust-encoding.

**`Encoding`** is a single character encoding.
It contains `encode` and `decode` methods for converting `String` to `Vec<u8>` and vice versa.
For the error handling, they receive **traps** (`EncoderCall` and `DecoderCall` respectively)
which replace any error with some string (e.g. `U+FFFD`) or sequence (e.g. `?`).
You can also use `EncodeStrict` and `DecodeStrict` traps to stop on an error.

There are two ways to get `Encoding`:

* `encoding::all` has static items for every supported encoding.
  You should use them when the encoding would not change or only handful of them are required.
  Combined with link-time optimization, any unused encoding would be discarded from the binary.
* `encoding::label` has functions to dynamically get an encoding from given string ("label").
  They will return a static reference to the encoding, which type is also known as `EncodingRef`.
  It is useful when a list of required encodings is not available in advance,
  but it will result in the larger binary and missed optimization opportunities.

**`Encoder`** is an experimental incremental encoder.
At each step of `raw_feed`, it receives a slice of string
and emits any encoded bytes to a generic `ByteWriter` (normally `Vec<u8>`).
It will stop at the first error if any, and would return a `CodecError` struct in that case.
The caller is responsible for calling `raw_finish` at the end of encoding process.

**`Decoder`** is an experimental incremental decoder.
At each step of `raw_feed`, it receives a slice of byte sequence
and emits any decoded characters to a generic `StringWriter` (normally `String`).
Otherwise it is identical to `Encoder`s.

One should prefer `Encoding::{encode,decode}` as a primary interface.
`Encoder` and `Decoder` is experimental and can change substantially.
See the additional documents on `encoding::types` module for more information on them.

## Supported Encodings

Rust-encoding covers all encodings specified by WHATWG Encoding Standard and some more:

* 7-bit strict ASCII (`ascii`)
* UTF-8 (`utf-8`)
* UTF-16 in little endian (`utf-16` or `utf-16le`) and big endian (`utf-16be`)
* All single byte encoding in WHATWG Encoding Standard:
    * IBM code page 866
    * ISO 8859-{2,3,4,5,6,7,8,10,13,14,15,16}
    * KOI8-R, KOI8-U
    * MacRoman (`macintosh`), Macintosh Cyrillic encoding (`x-mac-cyrillic`)
    * Windows code pages 874, 1250, 1251, 1252 (instead of ISO 8859-1), 1253,
      1254 (instead of ISO 8859-9), 1255, 1256, 1257, 1258
* All multi byte encodings in WHATWG Encoding Standard:
    * Windows code page 949 (`euc-kr`, since the strict EUC-KR is hardly used)
    * EUC-JP and Windows code page 932 (`shift_jis`,
      since it's the most widespread extension to Shift_JIS)
    * ISO-2022-JP with asymmetric JIS X 0212 support
    * GB 18030
    * HZ
    * Big5-2003 with HKSCS-2008 extensions
* ISO 8859-1 (distinct from Windows code page 1252)

Parenthesized names refer to the encoding's primary name assigned by WHATWG Encoding Standard.

Many legacy character encodings lack the proper specification,
and even those that have a specification are highly dependent of the actual implementation.
Consequently one should be careful when picking a desired character encoding.
The only standards reliable in this regard are WHATWG Encoding Standard and
[vendor-provided mappings from the Unicode consortium](http://www.unicode.org/Public/MAPPINGS/).
Whenever in doubt, look at the source code and specifications for detailed explanations.

*/

#![crate_name = "encoding"]
#![crate_type = "lib"]
#![comment = "Character encoding support for Rust"]
#![license = "MIT"]

#![feature(globs, macro_rules)]

#![allow(experimental)]

// XXX temporarily cope with the nightly, see #47
#![allow(unknown_features)]
#![feature(slicing_syntax)]

#[cfg(test)] extern crate test;

pub use self::types::{CodecError, ByteWriter, StringWriter,
                      Encoder, Decoder, EncodingRef, Encoding,
                      EncoderTrapFunc, DecoderTrapFunc, DecoderTrap,
                      DecodeStrict, DecodeReplace, DecodeIgnore,
                      EncoderTrap, EncodeStrict, EncodeReplace,
                      EncodeIgnore, EncodeNcrEscape, decode,
                      EncoderCall, DecoderCall}; // reexport

mod util;
#[cfg(test)] mod testutils;

pub mod types;

/// Indices used for character encoding implementation. Semi-internal.
#[unstable]
pub mod index {
    pub mod ibm866;
    pub mod iso_8859_2;
    pub mod iso_8859_3;
    pub mod iso_8859_4;
    pub mod iso_8859_5;
    pub mod iso_8859_6;
    pub mod iso_8859_7;
    pub mod iso_8859_8;
    pub mod iso_8859_10;
    pub mod iso_8859_13;
    pub mod iso_8859_14;
    pub mod iso_8859_15;
    pub mod iso_8859_16;
    pub mod koi8_r;
    pub mod koi8_u;
    pub mod macintosh;
    pub mod windows_874;
    pub mod windows_1250;
    pub mod windows_1251;
    pub mod windows_1252;
    pub mod windows_1253;
    pub mod windows_1254;
    pub mod windows_1255;
    pub mod windows_1256;
    pub mod windows_1257;
    pub mod windows_1258;
    pub mod x_mac_cyrillic;
    pub mod big5;
    pub mod euc_kr;
    pub mod gb18030;
    pub mod gb18030_ranges;
    pub mod jis0208;
    pub mod jis0212;
}

/// Codec implementations.
#[unstable]
pub mod codec {
    pub mod error;
    pub mod ascii;
    pub mod singlebyte;
    pub mod utf_8;
    pub mod utf_16;
    pub mod korean;
    pub mod japanese;
    pub mod simpchinese;
    pub mod tradchinese;
    pub mod whatwg;
}

pub mod all;
pub mod label;

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_decode() {
        fn test_one(input: &[u8], expected_result: &str, expected_encoding: &str) {
            let (result, used_encoding) = decode(
                input, DecodeStrict, all::ISO_8859_1 as EncodingRef);
            let result = result.unwrap();
            assert_eq!(used_encoding.name(), expected_encoding);
            assert_eq!(result[], expected_result);
        }

        test_one([0xEF, 0xBB, 0xBF, 0xC3, 0xA9], "é", "utf-8");
        test_one([0xC3, 0xA9], "Ã©", "iso-8859-1");

        test_one([0xFE, 0xFF, 0x00, 0xE9], "é", "utf-16be");
        test_one([0x00, 0xE9], "\x00é", "iso-8859-1");

        test_one([0xFF, 0xFE, 0xE9, 0x00], "é", "utf-16le");
        test_one([0xE9, 0x00], "é\x00", "iso-8859-1");
    }
}

<|endoftext|>


<commit_before>use regex::Regex;
use chrono::NaiveDate;
use reqwest;
use std::io::Read;
use select::document::Document;
use select::predicate::{Name, Attr};
use rustc_serialize::json::Json;
use errors::*;

// Some formats borrowed from https://github.com/amir/article-date-extractor
static FMTS: &'static [&str] = &["%A, %B %e, %Y", "%Y-%m-%dT%H:%M:%S%:z", "/%Y/%m/%d/", "/%Y/%d/%m/", "%Y-%m-%d", "%B %e, %Y", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ", "%B %k, %Y, %H:%M %p", "%Y-%m-%d %H:%M:%S.000000"];

// Use lazy_static to ensure we only compile the regex once
lazy_static! {
    // Regex by Newspaper3k  - https://github.com/codelucas/newspaper/blob/master/newspaper/urls.py
    static ref RE: Regex = Regex::new(r"([\./\-_]{0,1}(19|20)\d{2})[\./\-_]{0,1}(([0-3]{0,1}[0-9][\./\-_])|(\w{3,5}[\./\-_]))([0-3]{0,1}[0-9][\./\-]{0,1})?").unwrap();
}

// Parse the date, trying out each format
fn parse_date(input: &str) -> Result<NaiveDate> {
    let mut result: Result<NaiveDate> = Err("None of the formats matched the date".into());

    'outer: for fmt in FMTS {
        if let Ok(v) = NaiveDate::parse_from_str(input, fmt) {
            { result = Ok(v);
              break 'outer; }
        }
    }

    result
}

// Extract date from a URL
fn extract_from_url(url: &str) -> Option<String> {
    let mut date: Option<String> = None;

    if let Some(val) = RE.find(url) {
        date = Some(val.as_str().to_string());
    }

    date
}

// Extract date from JSON-LD
fn extract_from_ldjson<'a>(html: &'a Document) -> Option<String> {
    let mut json_date: Option<String> = None;
    let mut ldjson = String::new();
    if let Some(ldj) = html.find(Attr("type", "application/ld+json")).next() {
        ldjson = ldj.text();
    }

    let decoded_ldjson = Json::from_str(ldjson.as_str()).unwrap();

    if let Some(date_published) = decoded_ldjson.search("datePublished") {
        if let Some(date) = date_published.as_string() {
            json_date = Some(date.to_string())
        }
    } else if let Some(date_created) = decoded_ldjson.search("dateCreated") {
        if let Some(date) = date_created.as_string() {
            json_date = Some(date.to_string())
        }
    }

    json_date
}

// Extract date from meta tags
fn extract_from_meta<'a>(html: &'a Document) -> Option<String> {
    let mut meta_date: Option<String> = None;

    'outer: for meta in html.find(Name("meta")) {
        let meta_name     = meta.attr("name").unwrap_or("").to_lowercase();
        let item_prop     = meta.attr("itemprop").unwrap_or("").to_lowercase();
        let http_equiv    = meta.attr("http-equiv").unwrap_or("").to_lowercase();
        let meta_property = meta.attr("property").unwrap_or("").to_lowercase();

        match meta_name.as_ref() {
            "pubdate"               | "publishdate"                  | "timestamp"       |
            "dc.date.issued"        | "date"                         | "sailthru.date"   |
            "article.published"     | "published-date"               | "article.created" |
            "article_date_original" | "cxenseparse:recs:publishtime" | "date_published"  => { if let Some(ct) = meta.attr("content") {
                                                                                meta_date = Some(ct.trim().to_string())
                                                                                }
                                                                                break 'outer; },
            _ => {},
        }

        match item_prop.as_ref() {
            "datepublished" | "datecreated" => { if let Some(ct) = meta.attr("content") {
                                                   meta_date = Some(ct.trim().to_string())
                                                 }
                                                 break 'outer; },
            _ => {},
        }

        match http_equiv.as_ref() {
            "date" =>  { if let Some(ct) = meta.attr("content") {
                           meta_date = Some(ct.trim().to_string())
                         }
                         break 'outer; },
            _ => {},
        }

        match meta_property.as_ref() {
            "article:published_time" | "bt:pubdate" => { if let Some(ct) = meta.attr("content") {
                                                           meta_date = Some(ct.trim().to_string())
                                                         }
                                                         break 'outer; },
            "og:image"                              => { if let Some(url) = meta.attr("content") {
                                                           meta_date = extract_from_url(url.trim())
                                                         }
                                                         break 'outer; },

            _ => {},
        }

    }

    meta_date
}

// Extract from html tags
fn extract_from_html_tag<'a>(html: &'a Document) -> Option<String> {
    let mut date: Option<String> = None;

    for time in html.find(Name("time")) {
        if let Some(dt) = time.attr("datetime") {
            date = Some(dt.to_string())
        } else if let Some("timestamp") = time.attr("class") {
            date = Some(time.text())
        }
    }

    if date.is_none() {
        for tag in html.find(Name("meta")) {
            if let Some("datePublished") = tag.attr("itemprop") {
                if let Some(v) = tag.attr("content") {
                    date = Some(v.to_string())
                } else {
                    date = Some(tag.text())
                }
            }
        }
    }

    date
}

// Try to extract the date by using each function one by one
pub fn extract_article_published_date(link: &str, html: Option<String>) -> Result<NaiveDate> {
    let mut body: String = String::new();
    let mut parsed_body: Option<Document> = None;

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    }

    if html.is_none() {
        if let Ok(mut response) = reqwest::get(link) {
            response.read_to_string(&mut body).unwrap();
            let doc = Document::from(body.as_str());
            parsed_body = Some(doc);
        } else {
            return Err("Couldn't open the link".into())
        }
    } else {
        parsed_body = Some(Document::from(html.unwrap().as_str()))
    }

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_ldjson(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_meta(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_html_tag(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else {
        return Err("Couldn't find the date to parse".into())
    }
}

// Unit tests
#[cfg(test)]
mod test {
    use super::extract_from_url;
    use super::parse_date;
    use super::extract_from_meta;
    use super::extract_from_ldjson;
    use super::extract_from_html_tag;
    use super::extract_article_published_date;
    use chrono::NaiveDate;
    use reqwest;
    use std::string::String;
    use std::io::Read;
    use select::document::Document;

    #[test]
    fn parsing_date() {
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/11/30/").unwrap());
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/30/11/").unwrap());

        assert!(parse_date("bad_format").is_err());
    }

    #[test]
    fn extracting_from_url() {
        let link = "http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html";
        assert_eq!(Some("/2015/11/28/".to_string()), extract_from_url(link));

        let link = "";
        assert_eq!(None, extract_from_url(link));
    }

    #[test]
    fn extracting_from_meta() {
        let mut response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some(("2015-11-30 23:50:48".to_string())), extract_from_meta(&document));
    }

    #[test]
    fn extracting_from_ldjson() {
        let mut response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some("2015-12-01T07:50:48Z".to_string()), extract_from_ldjson(&document));
    }

    #[test]
    fn extracting_from_html_tag() {
        let mut response = reqwest::get("http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some("2015-11-29T00:44:59Z".to_string()), extract_from_html_tag(&document));
    }

    #[test]
    fn extracting_article_published_date() {
        let link = "http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html";
        let mut response = reqwest::get("http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();

        assert_eq!(NaiveDate::from_ymd(2015,11,28), extract_article_published_date(&link, None).unwrap());
        assert_eq!(NaiveDate::from_ymd(2015,11,28), extract_article_published_date(&link, Some(body)).unwrap());

        assert!((extract_article_published_date("", None)).is_err());
    }
}
<commit_msg>Adjust regex to not match short urls<commit_after>use regex::Regex;
use chrono::NaiveDate;
use reqwest;
use std::io::Read;
use select::document::Document;
use select::predicate::{Name, Attr};
use rustc_serialize::json::Json;
use errors::*;

// Some formats borrowed from https://github.com/amir/article-date-extractor
static FMTS: &'static [&str] = &["%A, %B %e, %Y", "%Y-%m-%dT%H:%M:%S%:z", "/%Y/%m/%d/", "/%Y/%d/%m/", "%Y-%m-%d", "%B %e, %Y", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ", "%B %k, %Y, %H:%M %p", "%Y-%m-%d %H:%M:%S.000000"];

// Use lazy_static to ensure we only compile the regex once
lazy_static! {
    // Regex by Newspaper3k  - https://github.com/codelucas/newspaper/blob/master/newspaper/urls.py
    static ref RE: Regex = Regex::new(r"([\./\-_]{0,1}(19|20)\d{2})[\./\-_]{0,1}(([0-3]{0,1}[0-9][\./\-_])|(\w{3,5}[\./\-_]))([0-3]{0,1}[0-9][\./\-]{0,1})").unwrap();
}

// Parse the date, trying out each format
fn parse_date(input: &str) -> Result<NaiveDate> {
    let mut result: Result<NaiveDate> = Err("None of the formats matched the date".into());

    'outer: for fmt in FMTS {
        if let Ok(v) = NaiveDate::parse_from_str(input, fmt) {
            { result = Ok(v);
              break 'outer; }
        }
    }

    result
}

// Extract date from a URL
fn extract_from_url(url: &str) -> Option<String> {
    let mut date: Option<String> = None;

    if let Some(val) = RE.find(url) {
        date = Some(val.as_str().to_string());
    }

    date
}

// Extract date from JSON-LD
fn extract_from_ldjson<'a>(html: &'a Document) -> Option<String> {
    let mut json_date: Option<String> = None;
    let mut ldjson = String::new();
    if let Some(ldj) = html.find(Attr("type", "application/ld+json")).next() {
        ldjson = ldj.text();
    }

    let decoded_ldjson = Json::from_str(ldjson.as_str()).unwrap();

    if let Some(date_published) = decoded_ldjson.search("datePublished") {
        if let Some(date) = date_published.as_string() {
            json_date = Some(date.to_string())
        }
    } else if let Some(date_created) = decoded_ldjson.search("dateCreated") {
        if let Some(date) = date_created.as_string() {
            json_date = Some(date.to_string())
        }
    }

    json_date
}

// Extract date from meta tags
fn extract_from_meta<'a>(html: &'a Document) -> Option<String> {
    let mut meta_date: Option<String> = None;

    'outer: for meta in html.find(Name("meta")) {
        let meta_name     = meta.attr("name").unwrap_or("").to_lowercase();
        let item_prop     = meta.attr("itemprop").unwrap_or("").to_lowercase();
        let http_equiv    = meta.attr("http-equiv").unwrap_or("").to_lowercase();
        let meta_property = meta.attr("property").unwrap_or("").to_lowercase();

        match meta_name.as_ref() {
            "pubdate"               | "publishdate"                  | "timestamp"       |
            "dc.date.issued"        | "date"                         | "sailthru.date"   |
            "article.published"     | "published-date"               | "article.created" |
            "article_date_original" | "cxenseparse:recs:publishtime" | "date_published"  => { if let Some(ct) = meta.attr("content") {
                                                                                meta_date = Some(ct.trim().to_string())
                                                                                }
                                                                                break 'outer; },
            _ => {},
        }

        match item_prop.as_ref() {
            "datepublished" | "datecreated" => { if let Some(ct) = meta.attr("content") {
                                                   meta_date = Some(ct.trim().to_string())
                                                 }
                                                 break 'outer; },
            _ => {},
        }

        match http_equiv.as_ref() {
            "date" =>  { if let Some(ct) = meta.attr("content") {
                           meta_date = Some(ct.trim().to_string())
                         }
                         break 'outer; },
            _ => {},
        }

        match meta_property.as_ref() {
            "article:published_time" | "bt:pubdate" => { if let Some(ct) = meta.attr("content") {
                                                           meta_date = Some(ct.trim().to_string())
                                                         }
                                                         break 'outer; },
            "og:image"                              => { if let Some(url) = meta.attr("content") {
                                                           meta_date = extract_from_url(url.trim())
                                                         }
                                                         break 'outer; },

            _ => {},
        }

    }

    meta_date
}

// Extract from html tags
fn extract_from_html_tag<'a>(html: &'a Document) -> Option<String> {
    let mut date: Option<String> = None;

    for time in html.find(Name("time")) {
        if let Some(dt) = time.attr("datetime") {
            date = Some(dt.to_string())
        } else if let Some("timestamp") = time.attr("class") {
            date = Some(time.text())
        }
    }

    if date.is_none() {
        for tag in html.find(Name("meta")) {
            if let Some("datePublished") = tag.attr("itemprop") {
                if let Some(v) = tag.attr("content") {
                    date = Some(v.to_string())
                } else {
                    date = Some(tag.text())
                }
            }
        }
    }

    date
}

// Try to extract the date by using each function one by one
pub fn extract_article_published_date(link: &str, html: Option<String>) -> Result<NaiveDate> {
    let mut body: String = String::new();
    let mut parsed_body: Option<Document> = None;

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    }

    if html.is_none() {
        if let Ok(mut response) = reqwest::get(link) {
            response.read_to_string(&mut body).unwrap();
            let doc = Document::from(body.as_str());
            parsed_body = Some(doc);
        } else {
            return Err("Couldn't open the link".into())
        }
    } else {
        parsed_body = Some(Document::from(html.unwrap().as_str()))
    }

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_ldjson(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_meta(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_html_tag(parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else {
        return Err("Couldn't find the date to parse".into())
    }
}

// Unit tests
#[cfg(test)]
mod test {
    use super::extract_from_url;
    use super::parse_date;
    use super::extract_from_meta;
    use super::extract_from_ldjson;
    use super::extract_from_html_tag;
    use super::extract_article_published_date;
    use chrono::NaiveDate;
    use reqwest;
    use std::string::String;
    use std::io::Read;
    use select::document::Document;

    #[test]
    fn parsing_date() {
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/11/30/").unwrap());
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/30/11/").unwrap());

        assert!(parse_date("bad_format").is_err());
    }

    #[test]
    fn extracting_from_url() {
        let link = "http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html";
        assert_eq!(Some("/2015/11/28/".to_string()), extract_from_url(link));

        let link = "";
        assert_eq!(None, extract_from_url(link));
    }

    #[test]
    fn extracting_from_meta() {
        let mut response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some(("2015-11-30 23:50:48".to_string())), extract_from_meta(&document));
    }

    #[test]
    fn extracting_from_ldjson() {
        let mut response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some("2015-12-01T07:50:48Z".to_string()), extract_from_ldjson(&document));
    }

    #[test]
    fn extracting_from_html_tag() {
        let mut response = reqwest::get("http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();
        let document = Document::from(body.as_str());

        assert_eq!(Some("2015-11-29T00:44:59Z".to_string()), extract_from_html_tag(&document));
    }

    #[test]
    fn extracting_article_published_date() {
        let link = "http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html";
        let mut response = reqwest::get("http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html").unwrap();
        let mut body = String::new();
        response.read_to_string(&mut body).unwrap();

        assert_eq!(NaiveDate::from_ymd(2015,11,28), extract_article_published_date(&link, None).unwrap());
        assert_eq!(NaiveDate::from_ymd(2015,11,28), extract_article_published_date(&link, Some(body)).unwrap());

        assert!((extract_article_published_date("", None)).is_err());
    }
}
<|endoftext|>


<commit_before>use syntax;
use syntax::codemap::DUMMY_SP;
use syntax::ext::base::{ ExtCtxt, DummyMacroLoader };

/// Create a fake ExtCtxt to perform macro quasiquotes outside of rustc plugins
pub fn with_fake_extctxt<T, F: Fn(&ExtCtxt) -> T>(f: F) -> T {
  let ps = syntax::parse::ParseSess::new();
  let mut fg_cfg = Vec::new();
  let mut loader = DummyMacroLoader;

  let mut cx = syntax::ext::base::ExtCtxt::new(&ps, Vec::new(),
    syntax::ext::expand::ExpansionConfig::default("rust-peg".to_string()),
    &mut fg_cfg,
    &mut loader,
  );

  cx.bt_push(syntax::codemap::ExpnInfo{
    call_site: DUMMY_SP,
    callee: syntax::codemap::NameAndSpan {
      format: syntax::codemap::MacroBang(syntax::parse::token::intern("")),
      span: None,
      allow_internal_unstable: false,
    }
  });

  f(&cx)
}
<commit_msg>Support ExtCtxt::new changes in nightly<commit_after>use syntax;
use syntax::codemap::DUMMY_SP;
use syntax::ext::base::{ ExtCtxt, DummyMacroLoader };

/// Create a fake ExtCtxt to perform macro quasiquotes outside of rustc plugins
pub fn with_fake_extctxt<T, F: Fn(&ExtCtxt) -> T>(f: F) -> T {
  let ps = syntax::parse::ParseSess::new();
  let mut loader = DummyMacroLoader;

  let mut cx = syntax::ext::base::ExtCtxt::new(&ps, Vec::new(),
    syntax::ext::expand::ExpansionConfig::default("rust-peg".to_string()),
    &mut loader,
  );

  cx.bt_push(syntax::codemap::ExpnInfo{
    call_site: DUMMY_SP,
    callee: syntax::codemap::NameAndSpan {
      format: syntax::codemap::MacroBang(syntax::parse::token::intern("")),
      span: None,
      allow_internal_unstable: false,
    }
  });

  f(&cx)
}
<|endoftext|>


<commit_before>use nom::{be_i8, be_i16, be_i32, be_i64, be_u8, be_u16, be_u32, be_u64, float, double};
use rusticata_macros::*;
use std::collections::HashMap;

pub type UOctet    = u8;
pub type UShort    = u16;
pub type ULong     = u32;
pub type ULongLong = u64;
pub type Timestamp = u64;

named!(pub short_string<&str>,
    do_parse!(
        length: be_u8             >>
        string: take_str!(length) >>
        (string)
    )
);

named!(pub long_string<&str>,
    do_parse!(
        length: be_u32            >>
        string: take_str!(length) >>
        (string)
    )
);

pub fn gen_short_string<'a>(x:(&'a mut [u8],usize), s: &str) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(
    x,
    gen_be_u8!(s.len() as u8) >>
    gen_slice!(s.as_bytes())
  )
}

pub fn gen_long_string<'a>(x:(&'a mut [u8],usize), s: &str) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(
    x,
    gen_be_u32!(s.len() as u32) >>
    gen_slice!(s.as_bytes())
  )
}

//a long string is a BE u32 followed by data. Maybe handle this in the state machine?

// Field names MUST start with a letter, '$' or '#' and may continue with letters, '$' or '#', digits, or
// underlines, to a maximum length of 128 characters.
// The server SHOULD validate field names and upon receiving an invalid field name, it SHOULD
// signal a connection exception with reply code 503 (syntax error).

#[derive(Clone,Debug,PartialEq)]
pub enum Value {
  Boolean(bool),
  ShortShortInt(i8),
  ShortShortUInt(u8),
  ShortInt(i16),
  ShortUInt(u16),
  LongInt(i32),
  LongUInt(u32),
  LongLongInt(i64),
  LongLongUInt(u64),
  Float(f32),
  Double(f64),
  Decimal(f32),
  ShortString(String),
  LongString(String),
  Array(Vec<Value>),
  Timestamp(u64),
  Table(HashMap<String,Value>),
  None
}

named!(pub value<Value>,
  switch!(map!(be_u8, |u| u as char),
    't' => call!(parse_boolean)          |
    //FIXME: the spec says b for i8, B for u8, but U for i16, u for u16, I for i32, i for u32, etc
    // is that right?
    'b' => call!(parse_short_short_int)  |
    'B' => call!(parse_short_short_uint) |
    'U' => call!(parse_short_int)        |
    'u' => call!(parse_short_uint)       |
    'I' => call!(parse_long_int)         |
    'i' => call!(parse_long_uint)        |
    'L' => call!(parse_long_long_int)    |
    'l' => call!(parse_long_long_uint)   |
    'f' => call!(parse_float)            |
    'd' => call!(parse_double)           |
    'D' => call!(parse_decimal)          |
    's' => call!(parse_short_string)     |
    'S' => call!(parse_long_string)      |
    'A' => call!(parse_array)            |
    'T' => call!(parse_timestamp)        |
    'F' => call!(parse_table)            |
    'V' => value!(Value::None)
  )
);

named!(pub field_name_value<(String, Value)>,
  tuple!(map!(short_string, |s:&str| s.to_string()), value)
);

named!(parse_boolean<Value>,
  map!(be_u8, |u| Value::Boolean(u != 0))
);

named!(parse_short_short_int<Value>,
  map!(be_i8, |i| Value::ShortShortInt(i))
);
named!(parse_short_short_uint<Value>,
  map!(be_u8, |i| Value::ShortShortUInt(i))
);
named!(parse_short_int<Value>,
  map!(be_i16, |i| Value::ShortInt(i))
);
named!(parse_short_uint<Value>,
  map!(be_u16, |i| Value::ShortUInt(i))
);
named!(parse_long_int<Value>,
  map!(be_i32, |i| Value::LongInt(i))
);
named!(parse_long_uint<Value>,
  map!(be_u32, |i| Value::LongUInt(i))
);
named!(parse_long_long_int<Value>,
  map!(be_i64, |i| Value::LongLongInt(i))
);
named!(parse_long_long_uint<Value>,
  map!(be_u64, |i| Value::LongLongUInt(i))
);
named!(parse_float<Value>,
  map!(float, |i| Value::Float(i))
);
named!(parse_double<Value>,
  map!(double, |i| Value::Double(i))
);
named!(parse_decimal<Value>,
  map!(float, |i| Value::Decimal(i))
);
named!(parse_short_string<Value>,
  map!(short_string, |s:&str| Value::ShortString(s.to_string()))
);

named!(parse_long_string<Value>,
  map!(long_string, |s:&str| Value::LongString(s.to_string()))
);

named!(parse_array<Value>,
  do_parse!(
    quantity: be_u32 >>
    //FIXME: the spec specifies a long int there, but a long uint for the table?
    vec: map!(count!(value, quantity as usize), |v| Value::Array(v)) >>
    (vec)
  )
);
named!(parse_timestamp<Value>,
  map!(be_u64, |i| Value::Timestamp(i))
);
named!(parse_table<Value>,
  do_parse!(
    //FIXME: the spec specifies a long uint there, but a long int for the array?
    quantity: be_u32 >>
    h: map!(flat_map!(take!(quantity as usize), many0!(field_name_value)), |v:Vec<(String,Value)>| {
      Value::Table(v.iter().cloned().collect())
    }) >>
    (h)
  )
);

named!(pub field_table<HashMap<String,Value>>,
  do_parse!(
    //FIXME: the spec specifies a long uint there, but a long int for the array?
    quantity: be_u32 >>
    h: map!(flat_map!(take!(quantity as usize), many0!(complete!(field_name_value))), |v:Vec<(String,Value)>| {
      v.iter().cloned().collect()
    }) >>
    (h)
  )
);

pub fn gen_value<'a>(x:(&'a mut [u8],usize), v: &Value) -> Result<(&'a mut [u8],usize),GenError> {
  match *v {
    Value::Boolean(ref b) => {
      do_gen!(x,
        gen_be_u8!('t' as u8) >>
        gen_be_u8!(*b as u8)
      )
    },
    /*
    Value::ShortShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('b' as u8) >>
        gen_be_i8!(i)
      )
    },
    */
    Value::ShortShortUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('B' as u8) >>
        gen_be_u8!(*i)
      )
    },
    /*
    Value::ShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('U' as u8) >>
        gen_be_i16!(i)
      )
    },
    */
    Value::ShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('u' as u8) >>
        gen_be_u16!(*i)
      )
    },
    /*
    Value::LongInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('I' as u8) >>
        gen_be_i32!(*i)
      )
    },
    */
    Value::LongUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('i' as u8) >>
        gen_be_u32!(*i)
      )
    },
    /*
    Value::LongLongInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('L' as u8) >>
        gen_be_i64!(*i)
      )
    },
    */
    Value::LongLongUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('l' as u8) >>
        gen_be_u64!(*i)
      )
    },
    /*
    Value::Float(f32),
    Value::Double(f64),
    Value::Decimal(f32),
    */
    Value::ShortString(ref s) => {
      do_gen!(x,
        gen_be_u8!('s' as u8) >>
        gen_short_string(&s)
      )
    },
    Value::LongString(ref s) => {
      do_gen!(x,
        gen_be_u8!('s' as u8) >>
        gen_long_string(&s)
      )
    },
    Value::Timestamp(ref i) => {
      do_gen!(x,
        gen_be_u8!('T' as u8) >>
        gen_be_u64!(*i)
      )
    },
    Value::Array(ref v) => {
      if let Ok((x1, index1)) = gen_be_u8!(x, 'A' as u8) {
        if let Ok((x2, index2)) = gen_many_ref!((x1, index1+4), v, gen_value) {
          if let Ok((x3,_)) = gen_be_u32!((x2, index1), index2 - index1 - 4) {
            Ok((x3, index2))
          } else {
            Err(GenError::CustomError(42))
          }
        } else {
          Err(GenError::CustomError(42))
        }
      } else {
        Err(GenError::CustomError(42))
      }
    },
    Value::Table(ref h) => {
      if let Ok((x1, index1)) = gen_be_u8!(x, 'F' as u8) {
        if let Ok((x2, index2)) = gen_many_ref!((x1, index1+4), h, gen_field_value) {
          if let Ok((x3,_)) = gen_be_u32!((x2, index1), index2 - index1 - 4) {
            Ok((x3, index2))
          } else {
            Err(GenError::CustomError(42))
          }
        } else {
          Err(GenError::CustomError(42))
        }
      } else {
        Err(GenError::CustomError(42))
      }
    },
    /*
    Value::None
    */
    _ => Err(GenError::CustomError(1))
  }
}

pub fn gen_bool<'a>(x:(&'a mut [u8],usize), b: &bool) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u8!(x, if *b {1} else {0})
}

pub fn gen_be_u8<'a>(x:(&'a mut [u8],usize), i: &u8) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u8!(x, *i)
}

pub fn gen_be_u16<'a>(x:(&'a mut [u8],usize), i: &u16) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u16!(x, *i)
}
pub fn gen_be_u32<'a>(x:(&'a mut [u8],usize), i: &u32) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u32!(x, *i)
}
pub fn gen_be_u64<'a>(x:(&'a mut [u8],usize), i: &u64) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u64!(x, *i)
}

pub fn gen_field_value<'a>(x:(&'a mut [u8],usize), kv: &(&String,&Value)) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(x,
    gen_short_string(kv.0) >>
    gen_value(kv.1)
  )
}

pub fn gen_field_table<'a>(x:(&'a mut [u8],usize), h: &HashMap<String,Value>) -> Result<(&'a mut [u8],usize),GenError> {
  if let Ok((x2, index2)) = gen_many_ref!((x.0, x.1+4), h, gen_field_value) {
    if let Ok((x3,_)) = gen_be_u32!((x2, x.1), index2 - x.1 - 4) {
      Ok((x3, index2))
    } else {
      Err(GenError::CustomError(42))
    }
  } else {
    Err(GenError::CustomError(42))
  }
}

pub fn gen_nothing<'a>(x:(&'a mut [u8],usize)) -> Result<(&'a mut [u8],usize),GenError> {
  Ok(x)
}

<commit_msg>long strings have a 'S', not a 's' as prefix<commit_after>use nom::{be_i8, be_i16, be_i32, be_i64, be_u8, be_u16, be_u32, be_u64, float, double};
use rusticata_macros::*;
use std::collections::HashMap;

pub type UOctet    = u8;
pub type UShort    = u16;
pub type ULong     = u32;
pub type ULongLong = u64;
pub type Timestamp = u64;

named!(pub short_string<&str>,
    do_parse!(
        length: be_u8             >>
        string: take_str!(length) >>
        (string)
    )
);

named!(pub long_string<&str>,
    do_parse!(
        length: be_u32            >>
        string: take_str!(length) >>
        (string)
    )
);

pub fn gen_short_string<'a>(x:(&'a mut [u8],usize), s: &str) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(
    x,
    gen_be_u8!(s.len() as u8) >>
    gen_slice!(s.as_bytes())
  )
}

pub fn gen_long_string<'a>(x:(&'a mut [u8],usize), s: &str) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(
    x,
    gen_be_u32!(s.len() as u32) >>
    gen_slice!(s.as_bytes())
  )
}

//a long string is a BE u32 followed by data. Maybe handle this in the state machine?

// Field names MUST start with a letter, '$' or '#' and may continue with letters, '$' or '#', digits, or
// underlines, to a maximum length of 128 characters.
// The server SHOULD validate field names and upon receiving an invalid field name, it SHOULD
// signal a connection exception with reply code 503 (syntax error).

#[derive(Clone,Debug,PartialEq)]
pub enum Value {
  Boolean(bool),
  ShortShortInt(i8),
  ShortShortUInt(u8),
  ShortInt(i16),
  ShortUInt(u16),
  LongInt(i32),
  LongUInt(u32),
  LongLongInt(i64),
  LongLongUInt(u64),
  Float(f32),
  Double(f64),
  Decimal(f32),
  ShortString(String),
  LongString(String),
  Array(Vec<Value>),
  Timestamp(u64),
  Table(HashMap<String,Value>),
  None
}

named!(pub value<Value>,
  switch!(map!(be_u8, |u| u as char),
    't' => call!(parse_boolean)          |
    //FIXME: the spec says b for i8, B for u8, but U for i16, u for u16, I for i32, i for u32, etc
    // is that right?
    'b' => call!(parse_short_short_int)  |
    'B' => call!(parse_short_short_uint) |
    'U' => call!(parse_short_int)        |
    'u' => call!(parse_short_uint)       |
    'I' => call!(parse_long_int)         |
    'i' => call!(parse_long_uint)        |
    'L' => call!(parse_long_long_int)    |
    'l' => call!(parse_long_long_uint)   |
    'f' => call!(parse_float)            |
    'd' => call!(parse_double)           |
    'D' => call!(parse_decimal)          |
    's' => call!(parse_short_string)     |
    'S' => call!(parse_long_string)      |
    'A' => call!(parse_array)            |
    'T' => call!(parse_timestamp)        |
    'F' => call!(parse_table)            |
    'V' => value!(Value::None)
  )
);

named!(pub field_name_value<(String, Value)>,
  tuple!(map!(short_string, |s:&str| s.to_string()), value)
);

named!(parse_boolean<Value>,
  map!(be_u8, |u| Value::Boolean(u != 0))
);

named!(parse_short_short_int<Value>,
  map!(be_i8, |i| Value::ShortShortInt(i))
);
named!(parse_short_short_uint<Value>,
  map!(be_u8, |i| Value::ShortShortUInt(i))
);
named!(parse_short_int<Value>,
  map!(be_i16, |i| Value::ShortInt(i))
);
named!(parse_short_uint<Value>,
  map!(be_u16, |i| Value::ShortUInt(i))
);
named!(parse_long_int<Value>,
  map!(be_i32, |i| Value::LongInt(i))
);
named!(parse_long_uint<Value>,
  map!(be_u32, |i| Value::LongUInt(i))
);
named!(parse_long_long_int<Value>,
  map!(be_i64, |i| Value::LongLongInt(i))
);
named!(parse_long_long_uint<Value>,
  map!(be_u64, |i| Value::LongLongUInt(i))
);
named!(parse_float<Value>,
  map!(float, |i| Value::Float(i))
);
named!(parse_double<Value>,
  map!(double, |i| Value::Double(i))
);
named!(parse_decimal<Value>,
  map!(float, |i| Value::Decimal(i))
);
named!(parse_short_string<Value>,
  map!(short_string, |s:&str| Value::ShortString(s.to_string()))
);

named!(parse_long_string<Value>,
  map!(long_string, |s:&str| Value::LongString(s.to_string()))
);

named!(parse_array<Value>,
  do_parse!(
    quantity: be_u32 >>
    //FIXME: the spec specifies a long int there, but a long uint for the table?
    vec: map!(count!(value, quantity as usize), |v| Value::Array(v)) >>
    (vec)
  )
);
named!(parse_timestamp<Value>,
  map!(be_u64, |i| Value::Timestamp(i))
);
named!(parse_table<Value>,
  do_parse!(
    //FIXME: the spec specifies a long uint there, but a long int for the array?
    quantity: be_u32 >>
    h: map!(flat_map!(take!(quantity as usize), many0!(field_name_value)), |v:Vec<(String,Value)>| {
      Value::Table(v.iter().cloned().collect())
    }) >>
    (h)
  )
);

named!(pub field_table<HashMap<String,Value>>,
  do_parse!(
    //FIXME: the spec specifies a long uint there, but a long int for the array?
    quantity: be_u32 >>
    h: map!(flat_map!(take!(quantity as usize), many0!(complete!(field_name_value))), |v:Vec<(String,Value)>| {
      v.iter().cloned().collect()
    }) >>
    (h)
  )
);

pub fn gen_value<'a>(x:(&'a mut [u8],usize), v: &Value) -> Result<(&'a mut [u8],usize),GenError> {
  match *v {
    Value::Boolean(ref b) => {
      do_gen!(x,
        gen_be_u8!('t' as u8) >>
        gen_be_u8!(*b as u8)
      )
    },
    /*
    Value::ShortShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('b' as u8) >>
        gen_be_i8!(i)
      )
    },
    */
    Value::ShortShortUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('B' as u8) >>
        gen_be_u8!(*i)
      )
    },
    /*
    Value::ShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('U' as u8) >>
        gen_be_i16!(i)
      )
    },
    */
    Value::ShortInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('u' as u8) >>
        gen_be_u16!(*i)
      )
    },
    /*
    Value::LongInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('I' as u8) >>
        gen_be_i32!(*i)
      )
    },
    */
    Value::LongUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('i' as u8) >>
        gen_be_u32!(*i)
      )
    },
    /*
    Value::LongLongInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('L' as u8) >>
        gen_be_i64!(*i)
      )
    },
    */
    Value::LongLongUInt(ref i) => {
      do_gen!(x,
        gen_be_u8!('l' as u8) >>
        gen_be_u64!(*i)
      )
    },
    /*
    Value::Float(f32),
    Value::Double(f64),
    Value::Decimal(f32),
    */
    Value::ShortString(ref s) => {
      do_gen!(x,
        gen_be_u8!('s' as u8) >>
        gen_short_string(&s)
      )
    },
    Value::LongString(ref s) => {
      do_gen!(x,
        gen_be_u8!('S' as u8) >>
        gen_long_string(&s)
      )
    },
    Value::Timestamp(ref i) => {
      do_gen!(x,
        gen_be_u8!('T' as u8) >>
        gen_be_u64!(*i)
      )
    },
    Value::Array(ref v) => {
      if let Ok((x1, index1)) = gen_be_u8!(x, 'A' as u8) {
        if let Ok((x2, index2)) = gen_many_ref!((x1, index1+4), v, gen_value) {
          if let Ok((x3,_)) = gen_be_u32!((x2, index1), index2 - index1 - 4) {
            Ok((x3, index2))
          } else {
            Err(GenError::CustomError(42))
          }
        } else {
          Err(GenError::CustomError(42))
        }
      } else {
        Err(GenError::CustomError(42))
      }
    },
    Value::Table(ref h) => {
      if let Ok((x1, index1)) = gen_be_u8!(x, 'F' as u8) {
        if let Ok((x2, index2)) = gen_many_ref!((x1, index1+4), h, gen_field_value) {
          if let Ok((x3,_)) = gen_be_u32!((x2, index1), index2 - index1 - 4) {
            Ok((x3, index2))
          } else {
            Err(GenError::CustomError(42))
          }
        } else {
          Err(GenError::CustomError(42))
        }
      } else {
        Err(GenError::CustomError(42))
      }
    },
    /*
    Value::None
    */
    _ => Err(GenError::CustomError(1))
  }
}

pub fn gen_bool<'a>(x:(&'a mut [u8],usize), b: &bool) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u8!(x, if *b {1} else {0})
}

pub fn gen_be_u8<'a>(x:(&'a mut [u8],usize), i: &u8) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u8!(x, *i)
}

pub fn gen_be_u16<'a>(x:(&'a mut [u8],usize), i: &u16) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u16!(x, *i)
}
pub fn gen_be_u32<'a>(x:(&'a mut [u8],usize), i: &u32) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u32!(x, *i)
}
pub fn gen_be_u64<'a>(x:(&'a mut [u8],usize), i: &u64) -> Result<(&'a mut [u8],usize),GenError> {
  gen_be_u64!(x, *i)
}

pub fn gen_field_value<'a>(x:(&'a mut [u8],usize), kv: &(&String,&Value)) -> Result<(&'a mut [u8],usize),GenError> {
  do_gen!(x,
    gen_short_string(kv.0) >>
    gen_value(kv.1)
  )
}

pub fn gen_field_table<'a>(x:(&'a mut [u8],usize), h: &HashMap<String,Value>) -> Result<(&'a mut [u8],usize),GenError> {
  if let Ok((x2, index2)) = gen_many_ref!((x.0, x.1+4), h, gen_field_value) {
    if let Ok((x3,_)) = gen_be_u32!((x2, x.1), index2 - x.1 - 4) {
      Ok((x3, index2))
    } else {
      Err(GenError::CustomError(42))
    }
  } else {
    Err(GenError::CustomError(42))
  }
}

pub fn gen_nothing<'a>(x:(&'a mut [u8],usize)) -> Result<(&'a mut [u8],usize),GenError> {
  Ok(x)
}

<|endoftext|>