<commit_before>// The MIT License (MIT)
// Copyright (c) 2016 Scott Lamb <slamb@slamb.org>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

extern crate futures;
extern crate futures_cpupool;
#[macro_use] extern crate log;
extern crate http_entity;
extern crate hyper;
extern crate libc;
extern crate mime;
extern crate time;

use futures::{Future, Sink, Stream};
use futures::stream::BoxStream;
use futures_cpupool::CpuPool;
use http_entity::Entity;
use hyper::Error;
use hyper::header;
use std::ops::Range;
use std::os::unix::io::AsRawFd;
use std::os::unix::fs::{FileExt, MetadataExt};
use std::sync::Arc;

/// A HTTP entity created from a `std::fs::File` which reads the file
/// chunk-by-chunk on a CpuPool.
#[derive(Clone)]
pub struct ChunkedReadFile<B, C> {
    inner: Arc<ChunkedReadFileInner>,
    phantom: ::std::marker::PhantomData<(B, C)>,
}

struct ChunkedReadFileInner {
    len: u64,
    inode: u64,
    mtime: i64,
    mtime_nsec: i64,
    content_type: ::mime::Mime,
    f: std::fs::File,
    pool: Option<CpuPool>,
}

impl<B, C> ChunkedReadFile<B, C> {
    /// Creates a new ChunkedReadFile.
    ///
    /// `read(2)` calls will be performed on the supplied `pool` so that they don't block the
    /// tokio reactor thread on local disk I/O. Note that `File::open` and this constructor
    /// (specifically, its call to `fstat(2)`) may also block, so they typically shouldn't be
    /// called on the tokio reactor either.
    pub fn new(file: ::std::fs::File, pool: Option<CpuPool>, content_type: ::mime::Mime)
               -> Result<Self, Error> {
        let m = file.metadata()?;
        Ok(ChunkedReadFile{
            inner: Arc::new(ChunkedReadFileInner{
                len: m.len(),
                inode: m.ino(),
                mtime: m.mtime(),
                mtime_nsec: m.mtime_nsec(),
                content_type: content_type,
                f: file,
                pool: pool,
            }),
            phantom: ::std::marker::PhantomData,
        })
    }
}

impl<B, C> Entity<B> for ChunkedReadFile<B, C>
where B: 'static + Send + From<BoxStream<C, Error>>,
      C: 'static + Send + From<Vec<u8>> {
    fn len(&self) -> u64 { self.inner.len }

    fn get_range(&self, range: Range<u64>) -> B {
        // This stream breaks apart the file into chunks of at most CHUNK_SIZE. This size is
        // a tradeoff between memory usage and thread handoffs.
        static CHUNK_SIZE: u64 = 65536;

        if range.end - range.start > CHUNK_SIZE {
            // Tell the operating system that this fd will be used to read the given range
            // sequentially. This may encourage it to do (more) readahead.
            let fadvise_ret = unsafe {
                ::libc::posix_fadvise(self.inner.f.as_raw_fd(), range.start as i64,
                                      (range.end - range.start) as i64, libc::POSIX_FADV_SEQUENTIAL)
            };
            if fadvise_ret != 0 {
                warn!("posix_fadvise failed: {}", ::std::io::Error::from_raw_os_error(fadvise_ret));
            }
        }

        let stream = ::futures::stream::unfold((range, self.inner.clone()), move |(left, inner)| {
            if left.start == left.end { return None }
            let chunk_size = ::std::cmp::min(CHUNK_SIZE, left.end - left.start) as usize;
            let mut chunk = Vec::with_capacity(chunk_size);
            unsafe { chunk.set_len(chunk_size) };
            let bytes_read = match inner.f.read_at(&mut chunk, left.start) {
                Err(e) => return Some(Err(e.into())),
                Ok(b) => b,
            };
            chunk.truncate(bytes_read);
            Some(Ok((chunk.into(), (left.start + bytes_read as u64 .. left.end, inner))))
        });

        let stream = match self.inner.pool {
            Some(ref p) => {
                let (snd, rcv) = ::futures::sync::mpsc::channel(0);
                p.spawn(snd.send_all(stream.then(|i| Ok(i)))
                           .map(|_| ())
                           .map_err(|_| ())).forget();
                rcv.map_err(|()| hyper::Error::Incomplete)
                   .and_then(|r| ::futures::future::result(r))
                   .boxed()
            },
            None => stream.boxed(),
        };
        stream.into()
    }

    fn add_headers(&self, h: &mut header::Headers) {
        h.set(header::ContentType(self.inner.content_type.clone()));
    }

    fn etag(&self) -> Option<header::EntityTag> {
        // This etag format is similar to Apache's, with more mtime precision.
        // The etag should change if the file is modified or replaced. The length is probably
        // redundant but doesn't harm anything.
        Some(header::EntityTag::strong(format!("{:x}:{:x}:{:x}:{:x}", self.inner.inode,
                                               self.inner.len, self.inner.mtime,
                                               self.inner.mtime_nsec)))
    }

    fn last_modified(&self) -> Option<header::HttpDate> {
        Some(header::HttpDate(time::at_utc(time::Timespec{
            sec: self.inner.mtime,
            nsec: self.inner.mtime_nsec as i32,
        })))
    }
}
<commit_msg>remove a couple unnecessary lines<commit_after>// The MIT License (MIT)
// Copyright (c) 2016 Scott Lamb <slamb@slamb.org>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

extern crate futures;
extern crate futures_cpupool;
#[macro_use] extern crate log;
extern crate http_entity;
extern crate hyper;
extern crate libc;
extern crate mime;
extern crate time;

use futures::{Sink, Stream};
use futures::stream::BoxStream;
use futures_cpupool::CpuPool;
use http_entity::Entity;
use hyper::Error;
use hyper::header;
use std::ops::Range;
use std::os::unix::io::AsRawFd;
use std::os::unix::fs::{FileExt, MetadataExt};
use std::sync::Arc;

/// A HTTP entity created from a `std::fs::File` which reads the file
/// chunk-by-chunk on a CpuPool.
#[derive(Clone)]
pub struct ChunkedReadFile<B, C> {
    inner: Arc<ChunkedReadFileInner>,
    phantom: ::std::marker::PhantomData<(B, C)>,
}

struct ChunkedReadFileInner {
    len: u64,
    inode: u64,
    mtime: i64,
    mtime_nsec: i64,
    content_type: ::mime::Mime,
    f: std::fs::File,
    pool: Option<CpuPool>,
}

impl<B, C> ChunkedReadFile<B, C> {
    /// Creates a new ChunkedReadFile.
    ///
    /// `read(2)` calls will be performed on the supplied `pool` so that they don't block the
    /// tokio reactor thread on local disk I/O. Note that `File::open` and this constructor
    /// (specifically, its call to `fstat(2)`) may also block, so they typically shouldn't be
    /// called on the tokio reactor either.
    pub fn new(file: ::std::fs::File, pool: Option<CpuPool>, content_type: ::mime::Mime)
               -> Result<Self, Error> {
        let m = file.metadata()?;
        Ok(ChunkedReadFile{
            inner: Arc::new(ChunkedReadFileInner{
                len: m.len(),
                inode: m.ino(),
                mtime: m.mtime(),
                mtime_nsec: m.mtime_nsec(),
                content_type: content_type,
                f: file,
                pool: pool,
            }),
            phantom: ::std::marker::PhantomData,
        })
    }
}

impl<B, C> Entity<B> for ChunkedReadFile<B, C>
where B: 'static + Send + From<BoxStream<C, Error>>,
      C: 'static + Send + From<Vec<u8>> {
    fn len(&self) -> u64 { self.inner.len }

    fn get_range(&self, range: Range<u64>) -> B {
        // This stream breaks apart the file into chunks of at most CHUNK_SIZE. This size is
        // a tradeoff between memory usage and thread handoffs.
        static CHUNK_SIZE: u64 = 65536;

        if range.end - range.start > CHUNK_SIZE {
            // Tell the operating system that this fd will be used to read the given range
            // sequentially. This may encourage it to do (more) readahead.
            let fadvise_ret = unsafe {
                ::libc::posix_fadvise(self.inner.f.as_raw_fd(), range.start as i64,
                                      (range.end - range.start) as i64, libc::POSIX_FADV_SEQUENTIAL)
            };
            if fadvise_ret != 0 {
                warn!("posix_fadvise failed: {}", ::std::io::Error::from_raw_os_error(fadvise_ret));
            }
        }

        let stream = ::futures::stream::unfold((range, self.inner.clone()), move |(left, inner)| {
            if left.start == left.end { return None }
            let chunk_size = ::std::cmp::min(CHUNK_SIZE, left.end - left.start) as usize;
            let mut chunk = Vec::with_capacity(chunk_size);
            unsafe { chunk.set_len(chunk_size) };
            let bytes_read = match inner.f.read_at(&mut chunk, left.start) {
                Err(e) => return Some(Err(e.into())),
                Ok(b) => b,
            };
            chunk.truncate(bytes_read);
            Some(Ok((chunk.into(), (left.start + bytes_read as u64 .. left.end, inner))))
        });

        let stream = match self.inner.pool {
            Some(ref p) => {
                let (snd, rcv) = ::futures::sync::mpsc::channel(0);
                p.spawn(snd.send_all(stream.then(|i| Ok(i))))
                 .forget();
                rcv.map_err(|()| hyper::Error::Incomplete)
                   .and_then(|r| ::futures::future::result(r))
                   .boxed()
            },
            None => stream.boxed(),
        };
        stream.into()
    }

    fn add_headers(&self, h: &mut header::Headers) {
        h.set(header::ContentType(self.inner.content_type.clone()));
    }

    fn etag(&self) -> Option<header::EntityTag> {
        // This etag format is similar to Apache's, with more mtime precision.
        // The etag should change if the file is modified or replaced. The length is probably
        // redundant but doesn't harm anything.
        Some(header::EntityTag::strong(format!("{:x}:{:x}:{:x}:{:x}", self.inner.inode,
                                               self.inner.len, self.inner.mtime,
                                               self.inner.mtime_nsec)))
    }

    fn last_modified(&self) -> Option<header::HttpDate> {
        Some(header::HttpDate(time::at_utc(time::Timespec{
            sec: self.inner.mtime,
            nsec: self.inner.mtime_nsec as i32,
        })))
    }
}
<|endoftext|>


<commit_before>use serde_json;
use gitlib::{GitRepo, GitRepositories};
use manifest_iter::ManifestIterator;

use std::path::{PathBuf, Path};
use std::fs::{File, DirBuilder};
use std::io::Write;
use std::collections::BTreeSet;

#[derive(Serialize, Deserialize, Debug)]
pub struct ManifestData {
    root_path: PathBuf,
    repositories: BTreeSet<PathBuf>,
}

impl ManifestData {
    fn empty(path: &Path) -> Self {
        ManifestData {
            repositories: BTreeSet::new(),
            root_path: path.to_path_buf(),
        }
    }

    fn add(&mut self, repo: &GitRepo) {
        let path_strip = match repo.path().strip_prefix(&self.root_path) {
            Ok(p) => p,
            Err(e) => {
                println!("Root path: {:?}", &self.root_path);
                println!("Path: {:?}", repo.path());
                println!("Error: {:?}", e);
                println!("##########################################");
                return;
            }
        };
        let path = PathBuf::from(path_strip.to_str().unwrap());
        self.repositories.insert(path);
    }

    pub fn root(&self) -> &Path {
        &self.root_path
    }

    pub fn repos(&self) -> &BTreeSet<PathBuf> {
        &self.repositories
    }
}

#[derive(Debug)]
pub enum ManifestError {
    BuildPath,
    PathNotFound,
}

#[derive(Debug)]
pub struct Manifest<'a> {
    path: &'a Path,
    data: ManifestData,
}

impl<'a> Manifest<'a> {
    pub fn open<P>(path: &'a P, root: &'a P) -> Result<Self, ManifestError>
        where P: AsRef<Path>
    {
        let path_ref = path.as_ref();

        let manifest_data = {
            let root_ref = root.as_ref();

            match path_ref.exists() {
                true => {
                    let file = File::open(path_ref).unwrap();

                    match serde_json::from_reader(&file) {
                        Ok(m) => m,
                        Err(_) => ManifestData::empty(root_ref),
                    }
                }
                false => ManifestData::empty(root_ref),
            }
        };

        Ok(Manifest {
               data: manifest_data,
               path: path_ref,
           })
    }

    pub fn add_repositories(&mut self, repos: GitRepositories) {
        for repo in repos {
            self.data.add(&repo);
        }

        let ser_data = serde_json::to_string_pretty(&self.data).unwrap();
        let mut file = self.get_file();

        match write!(file, "{}", ser_data) {
            Ok(_) => (),
            Err(e) => println!("{:#?}", e),
        }
    }

    fn get_file(&self) -> File {
        if !self.path.exists() {
            DirBuilder::new()
                .recursive(true)
                .create(self.path.parent().unwrap())
                .unwrap();
        }

        File::create(&self.path).unwrap()
    }

    pub fn paths(&self) -> ManifestIterator {
        ManifestIterator::new(&self.data)
    }
}<commit_msg>Tweaks<commit_after>use serde_json;
use gitlib::{GitRepo, GitRepositories};
use manifest_iter::ManifestIterator;

use std::path::{PathBuf, Path};
use std::fs::{File, DirBuilder};
use std::io::Write;
use std::collections::BTreeSet;

#[derive(Serialize, Deserialize, Debug)]
pub struct ManifestData {
    root_path: PathBuf,
    repositories: BTreeSet<PathBuf>,
}

impl ManifestData {
    fn empty(path: &Path) -> Self {
        ManifestData {
            repositories: BTreeSet::new(),
            root_path: path.to_path_buf(),
        }
    }

    fn add(&mut self, repo: &GitRepo) {
        let path_strip = match repo.path().strip_prefix(&self.root_path) {
            Ok(p) => p,
            Err(e) => {
                println!("Root path: {:?}", &self.root_path);
                println!("Path: {:?}", repo.path());
                println!("Error: {:?}", e);
                println!("##########################################");
                return;
            }
        };
        let path = PathBuf::from(path_strip.to_str().unwrap());
        self.repositories.insert(path);
    }

    pub fn root(&self) -> &Path {
        &self.root_path
    }

    pub fn repos(&self) -> &BTreeSet<PathBuf> {
        &self.repositories
    }
}

#[derive(Debug)]
pub enum ManifestError {
    BuildPath,
    PathNotFound,
}

#[derive(Debug)]
pub struct Manifest<'a> {
    path: &'a Path,
    data: ManifestData,
}

impl<'a> Manifest<'a> {
    pub fn open<P: AsRef<Path>>(path: &'a P, root: &'a P) -> Result<Self, ManifestError>
    {
        let path_ref = path.as_ref();

        let manifest_data = {
            let root_ref = root.as_ref();

            match path_ref.exists() {
                true => {
                    let file = File::open(path_ref).unwrap();

                    match serde_json::from_reader(&file) {
                        Ok(m) => m,
                        Err(_) => ManifestData::empty(root_ref),
                    }
                }
                false => ManifestData::empty(root_ref),
            }
        };

        Ok(Manifest {
               data: manifest_data,
               path: path_ref,
           })
    }

    pub fn add_repositories(&mut self, repos: GitRepositories) {
        for repo in repos {
            self.data.add(&repo);
        }

        let ser_data = serde_json::to_string_pretty(&self.data).unwrap();
        let mut file = self.get_file();

        match write!(file, "{}", ser_data) {
            Ok(_) => (),
            Err(e) => println!("{:#?}", e),
        }
    }

    fn get_file(&self) -> File {
        if !self.path.exists() {
            DirBuilder::new()
                .recursive(true)
                .create(self.path.parent().unwrap())
                .unwrap();
        }

        File::create(&self.path).unwrap()
    }

    pub fn paths(&self) -> ManifestIterator {
        ManifestIterator::new(&self.data)
    }

    pub fn path_in_manifest<P: AsRef<Path>>(&self, path: P) -> bool {
        true
    }
}<|endoftext|>


<commit_before>//! Base configuration options
//!
//! - `HEROKU`: Is this instance of cargo_registry currently running on Heroku.
//! - `S3_BUCKET`: The S3 bucket used to store crate files. If not present during development,
//!    cargo_registry will fall back to a local uploader.
//! - `S3_REGION`: The region in which the bucket was created. Optional if US standard.
//! - `AWS_ACCESS_KEY`: The access key to interact with S3. Optional if running a mirror.
//! - `AWS_SECRET_KEY`: The secret key to interact with S3. Optional if running a mirror.
//! - `S3_CDN`: Optional CDN configuration for building public facing URLs.

use crate::{env, uploaders::Uploader, Env};

pub struct Base {
    pub env: Env,
    uploader: Uploader,
}

impl Base {
    pub fn from_environment() -> Self {
        let heroku = dotenv::var("HEROKU").is_ok();
        let env = if heroku {
            Env::Production
        } else {
            Env::Development
        };

        let uploader = match env {
            Env::Production => {
                // `env` panics if these vars are not set, and in production for a primary instance,
                // that's what we want since we don't want to be able to start the server if the
                // server doesn't know where to upload crates.
                Self::s3_panic_if_missing_keys()
            }
            // In Development mode, either running as a primary instance or a read-only mirror
            _ => {
                if dotenv::var("S3_BUCKET").is_ok() {
                    // If we've set the `S3_BUCKET` variable to any value, use all of the values
                    // for the related S3 environment variables and configure the app to upload to
                    // and read from S3 like production does. All values except for bucket are
                    // optional, like production read-only mirrors.
                    println!("Using S3 uploader");
                    Self::s3_maybe_read_only()
                } else {
                    // If we don't set the `S3_BUCKET` variable, we'll use a development-only
                    // uploader that makes it possible to run and publish to a locally-running
                    // crates.io instance without needing to set up an account and a bucket in S3.
                    println!(
                        "Using local uploader, crate files will be in the local_uploads directory"
                    );
                    Uploader::Local
                }
            }
        };

        Self { env, uploader }
    }

    pub fn test() -> Self {
        let uploader = Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                String::from("alexcrichton-test"),
                None,
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                // When testing we route all API traffic over HTTP so we can
                // sniff/record it, but everywhere else we use https
                "http",
            )),
            index_bucket: Some(Box::new(s3::Bucket::new(
                String::from("alexcrichton-test"),
                None,
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                // When testing we route all API traffic over HTTP so we can
                // sniff/record it, but everywhere else we use https
                "http",
            ))),
            cdn: None,
        };
        Self {
            env: Env::Test,
            uploader,
        }
    }

    pub fn uploader(&self) -> &Uploader {
        &self.uploader
    }

    fn s3_panic_if_missing_keys() -> Uploader {
        let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
            Ok(name) => Some(Box::new(s3::Bucket::new(
                name,
                dotenv::var("S3_INDEX_REGION").ok(),
                env("AWS_ACCESS_KEY"),
                env("AWS_SECRET_KEY"),
                "https",
            ))),
            Err(_) => None,
        };
        Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                env("S3_BUCKET"),
                dotenv::var("S3_REGION").ok(),
                env("AWS_ACCESS_KEY"),
                env("AWS_SECRET_KEY"),
                "https",
            )),
            index_bucket,
            cdn: dotenv::var("S3_CDN").ok(),
        }
    }

    fn s3_maybe_read_only() -> Uploader {
        let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
            Ok(name) => Some(Box::new(s3::Bucket::new(
                name,
                dotenv::var("S3_INDEX_REGION").ok(),
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                "https",
            ))),
            Err(_) => None,
        };
        Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                env("S3_BUCKET"),
                dotenv::var("S3_REGION").ok(),
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                "https",
            )),
            index_bucket,
            cdn: dotenv::var("S3_CDN").ok(),
        }
    }
}
<commit_msg>config: Use `tracing` macros instead of `println!()`<commit_after>//! Base configuration options
//!
//! - `HEROKU`: Is this instance of cargo_registry currently running on Heroku.
//! - `S3_BUCKET`: The S3 bucket used to store crate files. If not present during development,
//!    cargo_registry will fall back to a local uploader.
//! - `S3_REGION`: The region in which the bucket was created. Optional if US standard.
//! - `AWS_ACCESS_KEY`: The access key to interact with S3. Optional if running a mirror.
//! - `AWS_SECRET_KEY`: The secret key to interact with S3. Optional if running a mirror.
//! - `S3_CDN`: Optional CDN configuration for building public facing URLs.

use crate::{env, uploaders::Uploader, Env};

pub struct Base {
    pub env: Env,
    uploader: Uploader,
}

impl Base {
    pub fn from_environment() -> Self {
        let heroku = dotenv::var("HEROKU").is_ok();
        let env = if heroku {
            Env::Production
        } else {
            Env::Development
        };

        let uploader = match env {
            Env::Production => {
                // `env` panics if these vars are not set, and in production for a primary instance,
                // that's what we want since we don't want to be able to start the server if the
                // server doesn't know where to upload crates.
                Self::s3_panic_if_missing_keys()
            }
            // In Development mode, either running as a primary instance or a read-only mirror
            _ => {
                if dotenv::var("S3_BUCKET").is_ok() {
                    // If we've set the `S3_BUCKET` variable to any value, use all of the values
                    // for the related S3 environment variables and configure the app to upload to
                    // and read from S3 like production does. All values except for bucket are
                    // optional, like production read-only mirrors.
                    info!("Using S3 uploader");
                    Self::s3_maybe_read_only()
                } else {
                    // If we don't set the `S3_BUCKET` variable, we'll use a development-only
                    // uploader that makes it possible to run and publish to a locally-running
                    // crates.io instance without needing to set up an account and a bucket in S3.
                    info!(
                        "Using local uploader, crate files will be in the local_uploads directory"
                    );
                    Uploader::Local
                }
            }
        };

        Self { env, uploader }
    }

    pub fn test() -> Self {
        let uploader = Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                String::from("alexcrichton-test"),
                None,
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                // When testing we route all API traffic over HTTP so we can
                // sniff/record it, but everywhere else we use https
                "http",
            )),
            index_bucket: Some(Box::new(s3::Bucket::new(
                String::from("alexcrichton-test"),
                None,
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                // When testing we route all API traffic over HTTP so we can
                // sniff/record it, but everywhere else we use https
                "http",
            ))),
            cdn: None,
        };
        Self {
            env: Env::Test,
            uploader,
        }
    }

    pub fn uploader(&self) -> &Uploader {
        &self.uploader
    }

    fn s3_panic_if_missing_keys() -> Uploader {
        let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
            Ok(name) => Some(Box::new(s3::Bucket::new(
                name,
                dotenv::var("S3_INDEX_REGION").ok(),
                env("AWS_ACCESS_KEY"),
                env("AWS_SECRET_KEY"),
                "https",
            ))),
            Err(_) => None,
        };
        Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                env("S3_BUCKET"),
                dotenv::var("S3_REGION").ok(),
                env("AWS_ACCESS_KEY"),
                env("AWS_SECRET_KEY"),
                "https",
            )),
            index_bucket,
            cdn: dotenv::var("S3_CDN").ok(),
        }
    }

    fn s3_maybe_read_only() -> Uploader {
        let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
            Ok(name) => Some(Box::new(s3::Bucket::new(
                name,
                dotenv::var("S3_INDEX_REGION").ok(),
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                "https",
            ))),
            Err(_) => None,
        };
        Uploader::S3 {
            bucket: Box::new(s3::Bucket::new(
                env("S3_BUCKET"),
                dotenv::var("S3_REGION").ok(),
                dotenv::var("AWS_ACCESS_KEY").unwrap_or_default(),
                dotenv::var("AWS_SECRET_KEY").unwrap_or_default(),
                "https",
            )),
            index_bucket,
            cdn: dotenv::var("S3_CDN").ok(),
        }
    }
}
<|endoftext|>


<commit_before>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("h5 a, h6 a, strong")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| Some(a.text_contents()))
        .ok_or(InsertError::HTML(()))
}

fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("img")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| {
            a.attributes.borrow().get("src").and_then(
                |src| Some(src.to_string()),
            )
        })
        .ok_or(InsertError::HTML(()))
}

fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    let selectors = vec![".userContent p", "span"];
    let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

    iters
        .map(|i| {
            i.fold(String::new(), |m, a| m + &a.as_node().to_string())
        })
        .filter(|i| !i.is_empty())
        .nth(0)
        .ok_or(InsertError::HTML(()))
}

fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
    Ok(
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .skip(1)
            .map(|a| {
                a.attributes.borrow().get("src").and_then(
                    |s| Some(s.to_string()),
                )
            })
            .filter(|s| s.is_some())
            .map(|s| s.unwrap())
            .collect::<Vec<String>>(),
    )
}

#[derive(AsChangeset, Debug)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
    title: String,
    message: String,
    html: String,
}

impl Images {
    fn from_ad(ad: &Ad, images: Vec<Uri>) -> Result<Images, InsertError> {
        let thumb = images
            .iter()
            .filter(|i| ad.thumbnail.contains(i.path()))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .nth(0);

        let mut rest = images.clone();
        if let Some(thumb) = thumb.clone() {
            rest.retain(|x| !thumb.contains(x.path()))
        };

        let collection = rest.iter()
            .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .collect::<Vec<String>>();

        let document = kuchiki::parse_html().one(ad.html.clone());
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    if let Some(i) = images.iter().find(|i| i.path() == u.path()) {
                        *x = ENDPOINT.to_string() + i.path().trim_left_matches('/');
                    } else {
                        *x = "".to_string();
                    }
                }
            };
        }

        let title = get_title(&document)?;
        let message = get_title(&document)?;
        Ok(Images {
            thumbnail: thumb,
            images: collection,
            title: title,
            html: document
                .select("div")
                .map_err(InsertError::HTML)?
                .nth(0)
                .ok_or(InsertError::HTML(()))?
                .as_node()
                .to_string(),
            message: message,
        })
    }
}

#[derive(Serialize, Queryable, Debug, Clone)]
pub struct Ad {
    pub id: String,
    pub html: String,
    pub political: i32,
    pub not_political: i32,
    pub title: String,
    pub message: String,
    pub thumbnail: String,
    pub created_at: DateTime<Utc>,
    pub updated_at: DateTime<Utc>,
    pub lang: String,
    pub images: Vec<String>,
    pub impressions: i32,
    pub political_probability: f64,
    pub targeting: Option<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // dropping images, but I can't see any way around it right now. Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it -- right now. This function swallows errors, and
    // there's a chance we'll end up with no images at the end, but I think we
    // can handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();
        let future = stream::iter_ok(self.image_urls())
            .map(|u| u.unwrap())
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            // currently we redownload images we already have, but ok.
            .filter(|u| {
                info!("testing {:?}", u.host());
                match u.host() {
                    Some(h) => (h == "pp-facebook-ads.s3.amazonaws.com" || h.ends_with("fbcdn.net")),
                    None => false
                }
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img.path());
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    if tuple.1.host().unwrap() != "pp-facebook-ads.s3.amazonaws.com" {
                        let credentials = DefaultCredentialsProvider::new()
                            .map_err(InsertError::AWS)?;
                        let tls = default_tls_client()
                            .map_err(InsertError::TLS)?;
                        let client = S3Client::new(tls, credentials, Region::UsEast1);
                        let req = PutObjectRequest {
                            bucket: "pp-facebook-ads".to_string(),
                            key: tuple.1.path().trim_left_matches('/').to_string(),
                            acl: Some("public-read".to_string()),
                            body: Some(tuple.0.to_vec()),
                            ..PutObjectRequest::default()
                        };
                        client.put_object(&req).map_err(InsertError::S3)?;
                    }
                    Ok(tuple.1)
                })
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    let update = Images::from_ad(&ad, imgs)?;
                    let connection = db.get().map_err(InsertError::Timeout)?;
                    diesel::update(ads.filter(id.eq(&ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(InsertError::DataBase)?;
                    info!("saved {:?}", ad.id);
                    Ok(())
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            });
        Box::new(future)
    }

    pub(self) fn image_urls(&self) -> Vec<Result<Uri, InsertError>> {
        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect()
    }

    pub fn get_ads_by_lang(
        language: &str,
        conn: &Pool<ConnectionManager<PgConnection>>,
    ) -> Result<Vec<Ad>, InsertError> {
        use schema::ads::dsl::*;
        let connection = conn.get().map_err(InsertError::Timeout)?;
        ads.filter(lang.eq(language))
            .filter(political_probability.gt(0.70))
            .order(updated_at.desc())
            .limit(100)
            .load::<Ad>(&*connection)
            .map_err(InsertError::DataBase)
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: &'a str,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    lang: &'a str,
    images: Vec<String>,
    impressions: i32,

    targeting: Option<String>,
}

impl<'a> NewAd<'a> {
    pub fn new(ad: &'a AdPost, lang: &'a str) -> Result<NewAd<'a>, InsertError> {
        info!("saving {}", ad.id);
        let document = kuchiki::parse_html().one(ad.html.clone());

        let thumb = get_image(&document)?;
        let images = get_images(&document)?;
        let message = get_message(&document)?;
        let title = get_title(&document)?;

        Ok(NewAd {
            id: &ad.id,
            html: &ad.html,
            // we try unwrapping or we chose the false branch in both of these
            // cases to count impressions
            political: if ad.political.unwrap_or(false) { 1 } else { 0 },
            not_political: if !ad.political.unwrap_or(true) { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            lang: lang,
            images: images,
            impressions: if !ad.political.is_some() { 1 } else { 0 },
            targeting: ad.targeting.clone(),
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;

        // increment impressions if this is a background save,
        // otherwise increment political counters
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                impressions.eq(
                    impressions + self.impressions,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;
        if self.targeting.is_some() && !ad.targeting.is_some() {
            diesel::update(ads.filter(id.eq(self.id)))
                .set(targeting.eq(self.targeting.clone()))
                .execute(&*connection)
                .map_err(InsertError::DataBase)?;
        };
        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: None,
            targeting: None,
        };
        let new_ad = NewAd::new(&post, "en-US").unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);

        assert_eq!(
            kuchiki::parse_html().one(new_ad.message).text_contents(),
            kuchiki::parse_html().one("
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
").text_contents()
        );
    }

    #[test]
    fn image_parsing() {
        use chrono::prelude::*;
        let ad = include_str!("./html-test.txt");
        let document = kuchiki::parse_html().one(ad);

        let saved_ad = Ad {
            id: "test".to_string(),
            html: ad.to_string(),
            political: 1,
            not_political: 2,
            title: get_title(&document).unwrap(),
            message: get_message(&document).unwrap(),
            thumbnail: get_image(&document).unwrap(),
            created_at: Utc::now(),
            updated_at: Utc::now(),
            lang: "US".to_string(),
            images: get_images(&document).unwrap(),
            impressions: 1,
            targeting: None,
            political_probability: 0.0,
        };
        let urls = saved_ad
            .image_urls()
            .into_iter()
            .map(|x| x.unwrap())
            .collect();
        let images = Images::from_ad(&saved_ad, urls).unwrap();
        assert!(images.html != saved_ad.html);
        assert!(!images.html.contains("fbcdn"));
        assert!(!images.html.contains("html"));
        assert!(images.images.len() == saved_ad.images.len());
        assert!(images.thumbnail.unwrap() != saved_ad.thumbnail);
    }
}
<commit_msg>spacing<commit_after>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("h5 a, h6 a, strong")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| Some(a.text_contents()))
        .ok_or(InsertError::HTML(()))
}

fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("img")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| {
            a.attributes.borrow().get("src").and_then(
                |src| Some(src.to_string()),
            )
        })
        .ok_or(InsertError::HTML(()))
}

fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    let selectors = vec![".userContent p", "span"];
    let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

    iters
        .map(|i| {
            i.fold(String::new(), |m, a| m + &a.as_node().to_string())
        })
        .filter(|i| !i.is_empty())
        .nth(0)
        .ok_or(InsertError::HTML(()))
}

fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
    Ok(
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .skip(1)
            .map(|a| {
                a.attributes.borrow().get("src").and_then(
                    |s| Some(s.to_string()),
                )
            })
            .filter(|s| s.is_some())
            .map(|s| s.unwrap())
            .collect::<Vec<String>>(),
    )
}

#[derive(AsChangeset, Debug)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
    title: String,
    message: String,
    html: String,
}

impl Images {
    fn from_ad(ad: &Ad, images: Vec<Uri>) -> Result<Images, InsertError> {
        let thumb = images
            .iter()
            .filter(|i| ad.thumbnail.contains(i.path()))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .nth(0);

        let mut rest = images.clone();
        if let Some(thumb) = thumb.clone() {
            rest.retain(|x| !thumb.contains(x.path()))
        };

        let collection = rest.iter()
            .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .collect::<Vec<String>>();

        let document = kuchiki::parse_html().one(ad.html.clone());
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    if let Some(i) = images.iter().find(|i| i.path() == u.path()) {
                        *x = ENDPOINT.to_string() + i.path().trim_left_matches('/');
                    } else {
                        *x = "".to_string();
                    }
                }
            };
        }

        let title = get_title(&document)?;
        let message = get_title(&document)?;
        Ok(Images {
            thumbnail: thumb,
            images: collection,
            title: title,
            html: document
                .select("div")
                .map_err(InsertError::HTML)?
                .nth(0)
                .ok_or(InsertError::HTML(()))?
                .as_node()
                .to_string(),
            message: message,
        })
    }
}

#[derive(Serialize, Queryable, Debug, Clone)]
pub struct Ad {
    pub id: String,
    pub html: String,
    pub political: i32,
    pub not_political: i32,
    pub title: String,
    pub message: String,
    pub thumbnail: String,
    pub created_at: DateTime<Utc>,
    pub updated_at: DateTime<Utc>,
    pub lang: String,
    pub images: Vec<String>,
    pub impressions: i32,
    pub political_probability: f64,
    pub targeting: Option<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // dropping images, but I can't see any way around it right now. Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it -- right now. This function swallows errors, and
    // there's a chance we'll end up with no images at the end, but I think we
    // can handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();
        let future = stream::iter_ok(self.image_urls())
            .map(|u| u.unwrap())
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            // currently we redownload images we already have, but ok.
            .filter(|u| {
                info!("testing {:?}", u.host());
                match u.host() {
                    Some(h) => (h == "pp-facebook-ads.s3.amazonaws.com" || h.ends_with("fbcdn.net")),
                    None => false
                }
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img.path());
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    if tuple.1.host().unwrap() != "pp-facebook-ads.s3.amazonaws.com" {
                        let credentials = DefaultCredentialsProvider::new()
                            .map_err(InsertError::AWS)?;
                        let tls = default_tls_client()
                            .map_err(InsertError::TLS)?;
                        let client = S3Client::new(tls, credentials, Region::UsEast1);
                        let req = PutObjectRequest {
                            bucket: "pp-facebook-ads".to_string(),
                            key: tuple.1.path().trim_left_matches('/').to_string(),
                            acl: Some("public-read".to_string()),
                            body: Some(tuple.0.to_vec()),
                            ..PutObjectRequest::default()
                        };
                        client.put_object(&req).map_err(InsertError::S3)?;
                    }
                    Ok(tuple.1)
                })
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    let update = Images::from_ad(&ad, imgs)?;
                    let connection = db.get().map_err(InsertError::Timeout)?;
                    diesel::update(ads.filter(id.eq(&ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(InsertError::DataBase)?;
                    info!("saved {:?}", ad.id);
                    Ok(())
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            });
        Box::new(future)
    }

    pub(self) fn image_urls(&self) -> Vec<Result<Uri, InsertError>> {
        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect()
    }

    pub fn get_ads_by_lang(
        language: &str,
        conn: &Pool<ConnectionManager<PgConnection>>,
    ) -> Result<Vec<Ad>, InsertError> {
        use schema::ads::dsl::*;
        let connection = conn.get().map_err(InsertError::Timeout)?;
        ads.filter(lang.eq(language))
            .filter(political_probability.gt(0.70))
            .order(updated_at.desc())
            .limit(100)
            .load::<Ad>(&*connection)
            .map_err(InsertError::DataBase)
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: &'a str,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    lang: &'a str,
    images: Vec<String>,
    impressions: i32,

    targeting: Option<String>,
}

impl<'a> NewAd<'a> {
    pub fn new(ad: &'a AdPost, lang: &'a str) -> Result<NewAd<'a>, InsertError> {
        info!("saving {}", ad.id);
        let document = kuchiki::parse_html().one(ad.html.clone());

        let thumb = get_image(&document)?;
        let images = get_images(&document)?;
        let message = get_message(&document)?;
        let title = get_title(&document)?;

        Ok(NewAd {
            id: &ad.id,
            html: &ad.html,
            // we try unwrapping or we chose the false branch in both of these
            // cases to count impressions
            political: if ad.political.unwrap_or(false) { 1 } else { 0 },
            not_political: if !ad.political.unwrap_or(true) { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            lang: lang,
            images: images,
            impressions: if !ad.political.is_some() { 1 } else { 0 },
            targeting: ad.targeting.clone(),
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;

        // increment impressions if this is a background save,
        // otherwise increment political counters
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                impressions.eq(
                    impressions + self.impressions,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;

        if self.targeting.is_some() && !ad.targeting.is_some() {
            diesel::update(ads.filter(id.eq(self.id)))
                .set(targeting.eq(self.targeting.clone()))
                .execute(&*connection)
                .map_err(InsertError::DataBase)?;
        };

        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: None,
            targeting: None,
        };
        let new_ad = NewAd::new(&post, "en-US").unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);

        assert_eq!(
            kuchiki::parse_html().one(new_ad.message).text_contents(),
            kuchiki::parse_html().one("
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
").text_contents()
        );
    }

    #[test]
    fn image_parsing() {
        use chrono::prelude::*;
        let ad = include_str!("./html-test.txt");
        let document = kuchiki::parse_html().one(ad);

        let saved_ad = Ad {
            id: "test".to_string(),
            html: ad.to_string(),
            political: 1,
            not_political: 2,
            title: get_title(&document).unwrap(),
            message: get_message(&document).unwrap(),
            thumbnail: get_image(&document).unwrap(),
            created_at: Utc::now(),
            updated_at: Utc::now(),
            lang: "US".to_string(),
            images: get_images(&document).unwrap(),
            impressions: 1,
            targeting: None,
            political_probability: 0.0,
        };
        let urls = saved_ad
            .image_urls()
            .into_iter()
            .map(|x| x.unwrap())
            .collect();
        let images = Images::from_ad(&saved_ad, urls).unwrap();
        assert!(images.html != saved_ad.html);
        assert!(!images.html.contains("fbcdn"));
        assert!(!images.html.contains("html"));
        assert!(images.images.len() == saved_ad.images.len());
        assert!(images.thumbnail.unwrap() != saved_ad.thumbnail);
    }
}
<|endoftext|>


<commit_before>use std::ptr::null;
use std::str::raw;
use libc::{c_int, c_char, c_double};
use sync::mutex::{StaticMutex, MUTEX_INIT};

#[link(name="gdal")]
extern {
    fn OGRRegisterAll();
    fn OGROpen(pszName: *c_char, bUpdate: c_int, pahDriverList: *()) -> *();
    fn OGR_DS_GetLayerCount(hDS: *()) -> c_int;
    fn OGR_DS_Destroy(hDataSource: *());
    fn OGR_DS_GetLayer(hDS: *(), iLayer: c_int) -> *();
    fn OGR_L_GetLayerDefn(hLayer: *()) -> *();
    fn OGR_L_GetNextFeature(hLayer: *()) -> *();
    fn OGR_FD_GetFieldCount(hDefn: *()) -> c_int;
    fn OGR_FD_GetFieldDefn(hDefn: *(), iField: c_int) -> *();
    fn OGR_F_GetFieldIndex(hFeat: *(), pszName: *c_char) -> c_int;
    fn OGR_F_GetFieldDefnRef(hFeat: *(), i: c_int) -> *();
    fn OGR_F_GetFieldAsString(hFeat: *(), iField: c_int) -> *c_char;
    fn OGR_F_GetFieldAsDouble(hFeat: *(), iField: c_int) -> c_double;
    fn OGR_F_GetGeometryRef(hFeat: *()) -> *();
    fn OGR_F_Destroy(hFeat: *());
    fn OGR_G_ExportToWkt(hGeom: *(), ppszSrcText: **c_char) -> c_int;
    fn OGR_G_ExportToJson(hGeometry: *()) -> *c_char;
    fn OGR_Fld_GetNameRef(hDefn: *()) -> *c_char;
    fn OGR_Fld_GetType(hDefn: *()) -> c_int;
    fn OGRFree(ptr: *());
    fn VSIFree(ptr: *());
}

static OFTInteger:        c_int = 0;
static OFTIntegerList:    c_int = 1;
static OFTReal:           c_int = 2;
static OFTRealList:       c_int = 3;
static OFTString:         c_int = 4;
static OFTStringList:     c_int = 5;
static OFTWideString:     c_int = 6;
static OFTWideStringList: c_int = 7;
static OFTBinary:         c_int = 8;
static OFTDate:           c_int = 9;
static OFTTime:           c_int = 10;
static OFTDateTime:       c_int = 11;

static mut LOCK: StaticMutex = MUTEX_INIT;
static mut registered_drivers: bool = false;

fn register_drivers() {
    unsafe {
        let _g = LOCK.lock();
        if ! registered_drivers {
            OGRRegisterAll();
            registered_drivers = true;
        }
    }
}

pub struct VectorDataset {
    c_dataset: *(),
}

impl VectorDataset {
    pub fn layer_count(&self) -> int {
        return unsafe { OGR_DS_GetLayerCount(self.c_dataset) } as int;
    }

    pub fn layer<'a>(&'a self, idx: int) -> Option<Layer<'a>> {
        let c_layer = unsafe { OGR_DS_GetLayer(self.c_dataset, idx as c_int) };
        return match c_layer.is_null() {
            true  => None,
            false => Some(Layer{vector_dataset: self, c_layer: c_layer}),
        };
    }
}

impl Drop for VectorDataset {
    fn drop(&mut self) {
        unsafe { OGR_DS_Destroy(self.c_dataset); }
    }
}

pub struct Layer<'a> {
    vector_dataset: &'a VectorDataset,
    c_layer: *(),
}

impl<'a> Layer<'a> {
    pub fn fields<'a>(&'a self) -> FieldIterator<'a> {
        let c_feature_defn = unsafe { OGR_L_GetLayerDefn(self.c_layer) };
        let total = unsafe { OGR_FD_GetFieldCount(c_feature_defn) } as int;
        return FieldIterator{
            layer: self,
            c_feature_defn: c_feature_defn,
            next_id: 0,
            total: total
        };
    }

    pub fn features<'a>(&'a self) -> FeatureIterator<'a> {
        return FeatureIterator{layer: self};
    }
}

pub struct FieldIterator<'a> {
    layer: &'a Layer<'a>,
    c_feature_defn: *(),
    next_id: int,
    total: int,
}

impl<'a> Iterator<Field<'a>> for FieldIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Field<'a>> {
        if self.next_id == self.total {
            return None;
        }
        let field = Field{
            layer: self.layer,
            c_field_defn: unsafe { OGR_FD_GetFieldDefn(
                self.c_feature_defn,
                self.next_id as c_int
            ) }
        };
        self.next_id += 1;
        return Some(field);
    }
}

pub struct Field<'a> {
    layer: &'a Layer<'a>,
    c_field_defn: *(),
}

impl<'a> Field<'a> {
    pub fn name(&'a self) -> String {
        unsafe { raw::from_c_str(OGR_Fld_GetNameRef(self.c_field_defn)) }
    }
}

pub struct FeatureIterator<'a> {
    layer: &'a Layer<'a>,
}

impl<'a> Iterator<Feature<'a>> for FeatureIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Feature<'a>> {
        let c_feature = unsafe { OGR_L_GetNextFeature(self.layer.c_layer) };
        return match c_feature.is_null() {
            true  => None,
            false => Some(Feature{layer: self.layer, c_feature: c_feature}),
        };
    }
}

pub struct Feature<'a> {
    layer: &'a Layer<'a>,
    c_feature: *(),
}

impl<'a> Feature<'a> {
    pub fn field(&self, name: String) -> Option<FieldValue> {
        return name.with_c_str(|c_name| unsafe {
            let field_id = OGR_F_GetFieldIndex(self.c_feature, c_name);
            if field_id == -1 {
                return None;
            }
            let field_defn = OGR_F_GetFieldDefnRef(self.c_feature, field_id);
            let field_type = OGR_Fld_GetType(field_defn);
            return match field_type {
                OFTString => {
                    let rv = OGR_F_GetFieldAsString(self.c_feature, field_id);
                    return Some(StringValue(raw::from_c_str(rv)));
                },
                OFTReal => {
                    let rv = OGR_F_GetFieldAsDouble(self.c_feature, field_id);
                    return Some(RealValue(rv as f64));
                },
                _ => fail!("Unknown field type {}", field_type)
            }
        });
    }

    pub fn wkt(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_wkt: *c_char = null();
            OGR_G_ExportToWkt(c_geom, &c_wkt);
            let wkt = raw::from_c_str(c_wkt);
            OGRFree(c_wkt as *());
            return wkt;
        }
    }

    pub fn json(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_json = OGR_G_ExportToJson(c_geom);
            let json = raw::from_c_str(c_json);
            VSIFree(c_json as *());
            return json;
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Feature<'a> {
    fn drop(&mut self) {
        unsafe { OGR_F_Destroy(self.c_feature); }
    }
}

pub fn open(path: &Path) -> Option<VectorDataset> {
    register_drivers();
    let filename = path.as_str().unwrap();
    let c_dataset = filename.with_c_str(|c_filename| {
        return unsafe { OGROpen(c_filename, 0, null()) };
    });
    return match c_dataset.is_null() {
        true  => None,
        false => Some(VectorDataset{c_dataset: c_dataset}),
    };
}

pub enum FieldValue {
    StringValue(String),
    RealValue(f64),
}

impl FieldValue {
    pub fn as_string(self) -> String {
        match self { StringValue(rv) => rv, _ => fail!("not a StringValue") }
    }

    pub fn as_real(self) -> f64 {
        match self { RealValue(rv) => rv, _ => fail!("not a RealValue") }
    }
}

#[cfg(test)]
mod test {
    use std::os::getenv;
    use std::path::Path;
    use super::{Feature, FeatureIterator, open};

    fn fixture_path(name: &str) -> Path {
        let envvar = "RUST_GDAL_TEST_FIXTURES";
        let fixtures = match getenv(envvar) {
            Some(p) => Path::new(p),
            None => fail!("Environment variable {} not set", envvar)
        };
        let rv = fixtures.join(name);
        return rv;
    }

    fn assert_almost_eq(a: f64, b: f64) {
        let f: f64 = a / b;
        assert!(f < 1.00001);
        assert!(f > 0.99999);
    }

    #[test]
    fn test_layer_count() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        assert_eq!(ds.layer_count(), 1);
    }

    fn with_features(fixture: &str, f: |FeatureIterator|) {
        let ds = open(&fixture_path(fixture)).unwrap();
        let layer = ds.layer(0).unwrap();
        f(layer.features());
    }

    fn with_first_feature(fixture: &str, f: |Feature|) {
        with_features(fixture, |mut features| f(features.next().unwrap()));
    }

    #[test]
    fn test_iterate_features() {
        with_features("roads.geojson", |mut features| {
            let feature_vec: Vec<Feature> = features.collect();
            assert_eq!(feature_vec.len(), 21);
        });
    }

    #[test]
    fn test_string_field() {
        with_features("roads.geojson", |mut features| {
            let feature = features.next().unwrap();
            assert_eq!(feature.field("highway".to_string())
                              .unwrap()
                              .as_string(),
                       "footway".to_string());
            assert_eq!(
                features.count(|field| {
                    let highway = field.field("highway".to_string())
                                       .unwrap()
                                       .as_string();
                    highway == "residential".to_string() }),
                2);
        });
    }

    #[test]
    fn test_float_field() {
        with_first_feature("roads.geojson", |feature| {
            assert_almost_eq(
                feature.field("sort_key".to_string())
                       .unwrap()
                       .as_real(),
                -9.0
            );
        });
    }

    #[test]
    fn test_missing_field() {
        with_first_feature("roads.geojson", |feature| {
            assert!(feature.field("no such field".to_string()).is_none());
        });
    }

    #[test]
    fn test_wkt() {
        with_first_feature("roads.geojson", |feature| {
            let wkt = feature.wkt();
            let wkt_ok = format!("{}{}",
                "LINESTRING (26.1019276 44.4302748,",
                "26.1019382 44.4303191,26.1020002 44.4304202)"
                ).to_string();
            assert_eq!(wkt, wkt_ok);
        });
    }

    #[test]
    fn test_json() {
        with_first_feature("roads.geojson", |feature| {
            let json = feature.json();
            let json_ok = format!("{}{}{}{}",
                "{ \"type\": \"LineString\", \"coordinates\": [ ",
                "[ 26.1019276, 44.4302748 ], ",
                "[ 26.1019382, 44.4303191 ], ",
                "[ 26.1020002, 44.4304202 ] ] }"
                ).to_string();
            assert_eq!(json, json_ok);
        });
    }

    #[test]
    fn test_schema() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.layer(0).unwrap();
        let name_list: Vec<String> = layer
            .fields()
            .map(|f| f.name())
            .collect();
        let ok_names = vec!("kind", "sort_key", "is_link", "is_tunnel",
                        "is_bridge", "railway", "highway")
                       .iter().map(|s| s.to_string()).collect();
        assert_eq!(name_list, ok_names);
    }
}
<commit_msg>`.count()` doesn't accept an argument any more<commit_after>use std::ptr::null;
use std::str::raw;
use libc::{c_int, c_char, c_double};
use sync::mutex::{StaticMutex, MUTEX_INIT};

#[link(name="gdal")]
extern {
    fn OGRRegisterAll();
    fn OGROpen(pszName: *c_char, bUpdate: c_int, pahDriverList: *()) -> *();
    fn OGR_DS_GetLayerCount(hDS: *()) -> c_int;
    fn OGR_DS_Destroy(hDataSource: *());
    fn OGR_DS_GetLayer(hDS: *(), iLayer: c_int) -> *();
    fn OGR_L_GetLayerDefn(hLayer: *()) -> *();
    fn OGR_L_GetNextFeature(hLayer: *()) -> *();
    fn OGR_FD_GetFieldCount(hDefn: *()) -> c_int;
    fn OGR_FD_GetFieldDefn(hDefn: *(), iField: c_int) -> *();
    fn OGR_F_GetFieldIndex(hFeat: *(), pszName: *c_char) -> c_int;
    fn OGR_F_GetFieldDefnRef(hFeat: *(), i: c_int) -> *();
    fn OGR_F_GetFieldAsString(hFeat: *(), iField: c_int) -> *c_char;
    fn OGR_F_GetFieldAsDouble(hFeat: *(), iField: c_int) -> c_double;
    fn OGR_F_GetGeometryRef(hFeat: *()) -> *();
    fn OGR_F_Destroy(hFeat: *());
    fn OGR_G_ExportToWkt(hGeom: *(), ppszSrcText: **c_char) -> c_int;
    fn OGR_G_ExportToJson(hGeometry: *()) -> *c_char;
    fn OGR_Fld_GetNameRef(hDefn: *()) -> *c_char;
    fn OGR_Fld_GetType(hDefn: *()) -> c_int;
    fn OGRFree(ptr: *());
    fn VSIFree(ptr: *());
}

static OFTInteger:        c_int = 0;
static OFTIntegerList:    c_int = 1;
static OFTReal:           c_int = 2;
static OFTRealList:       c_int = 3;
static OFTString:         c_int = 4;
static OFTStringList:     c_int = 5;
static OFTWideString:     c_int = 6;
static OFTWideStringList: c_int = 7;
static OFTBinary:         c_int = 8;
static OFTDate:           c_int = 9;
static OFTTime:           c_int = 10;
static OFTDateTime:       c_int = 11;

static mut LOCK: StaticMutex = MUTEX_INIT;
static mut registered_drivers: bool = false;

fn register_drivers() {
    unsafe {
        let _g = LOCK.lock();
        if ! registered_drivers {
            OGRRegisterAll();
            registered_drivers = true;
        }
    }
}

pub struct VectorDataset {
    c_dataset: *(),
}

impl VectorDataset {
    pub fn layer_count(&self) -> int {
        return unsafe { OGR_DS_GetLayerCount(self.c_dataset) } as int;
    }

    pub fn layer<'a>(&'a self, idx: int) -> Option<Layer<'a>> {
        let c_layer = unsafe { OGR_DS_GetLayer(self.c_dataset, idx as c_int) };
        return match c_layer.is_null() {
            true  => None,
            false => Some(Layer{vector_dataset: self, c_layer: c_layer}),
        };
    }
}

impl Drop for VectorDataset {
    fn drop(&mut self) {
        unsafe { OGR_DS_Destroy(self.c_dataset); }
    }
}

pub struct Layer<'a> {
    vector_dataset: &'a VectorDataset,
    c_layer: *(),
}

impl<'a> Layer<'a> {
    pub fn fields<'a>(&'a self) -> FieldIterator<'a> {
        let c_feature_defn = unsafe { OGR_L_GetLayerDefn(self.c_layer) };
        let total = unsafe { OGR_FD_GetFieldCount(c_feature_defn) } as int;
        return FieldIterator{
            layer: self,
            c_feature_defn: c_feature_defn,
            next_id: 0,
            total: total
        };
    }

    pub fn features<'a>(&'a self) -> FeatureIterator<'a> {
        return FeatureIterator{layer: self};
    }
}

pub struct FieldIterator<'a> {
    layer: &'a Layer<'a>,
    c_feature_defn: *(),
    next_id: int,
    total: int,
}

impl<'a> Iterator<Field<'a>> for FieldIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Field<'a>> {
        if self.next_id == self.total {
            return None;
        }
        let field = Field{
            layer: self.layer,
            c_field_defn: unsafe { OGR_FD_GetFieldDefn(
                self.c_feature_defn,
                self.next_id as c_int
            ) }
        };
        self.next_id += 1;
        return Some(field);
    }
}

pub struct Field<'a> {
    layer: &'a Layer<'a>,
    c_field_defn: *(),
}

impl<'a> Field<'a> {
    pub fn name(&'a self) -> String {
        unsafe { raw::from_c_str(OGR_Fld_GetNameRef(self.c_field_defn)) }
    }
}

pub struct FeatureIterator<'a> {
    layer: &'a Layer<'a>,
}

impl<'a> Iterator<Feature<'a>> for FeatureIterator<'a> {
    #[inline]
    fn next(&mut self) -> Option<Feature<'a>> {
        let c_feature = unsafe { OGR_L_GetNextFeature(self.layer.c_layer) };
        return match c_feature.is_null() {
            true  => None,
            false => Some(Feature{layer: self.layer, c_feature: c_feature}),
        };
    }
}

pub struct Feature<'a> {
    layer: &'a Layer<'a>,
    c_feature: *(),
}

impl<'a> Feature<'a> {
    pub fn field(&self, name: String) -> Option<FieldValue> {
        return name.with_c_str(|c_name| unsafe {
            let field_id = OGR_F_GetFieldIndex(self.c_feature, c_name);
            if field_id == -1 {
                return None;
            }
            let field_defn = OGR_F_GetFieldDefnRef(self.c_feature, field_id);
            let field_type = OGR_Fld_GetType(field_defn);
            return match field_type {
                OFTString => {
                    let rv = OGR_F_GetFieldAsString(self.c_feature, field_id);
                    return Some(StringValue(raw::from_c_str(rv)));
                },
                OFTReal => {
                    let rv = OGR_F_GetFieldAsDouble(self.c_feature, field_id);
                    return Some(RealValue(rv as f64));
                },
                _ => fail!("Unknown field type {}", field_type)
            }
        });
    }

    pub fn wkt(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_wkt: *c_char = null();
            OGR_G_ExportToWkt(c_geom, &c_wkt);
            let wkt = raw::from_c_str(c_wkt);
            OGRFree(c_wkt as *());
            return wkt;
        }
    }

    pub fn json(&self) -> String {
        unsafe {
            let c_geom = OGR_F_GetGeometryRef(self.c_feature);
            let c_json = OGR_G_ExportToJson(c_geom);
            let json = raw::from_c_str(c_json);
            VSIFree(c_json as *());
            return json;
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Feature<'a> {
    fn drop(&mut self) {
        unsafe { OGR_F_Destroy(self.c_feature); }
    }
}

pub fn open(path: &Path) -> Option<VectorDataset> {
    register_drivers();
    let filename = path.as_str().unwrap();
    let c_dataset = filename.with_c_str(|c_filename| {
        return unsafe { OGROpen(c_filename, 0, null()) };
    });
    return match c_dataset.is_null() {
        true  => None,
        false => Some(VectorDataset{c_dataset: c_dataset}),
    };
}

pub enum FieldValue {
    StringValue(String),
    RealValue(f64),
}

impl FieldValue {
    pub fn as_string(self) -> String {
        match self { StringValue(rv) => rv, _ => fail!("not a StringValue") }
    }

    pub fn as_real(self) -> f64 {
        match self { RealValue(rv) => rv, _ => fail!("not a RealValue") }
    }
}

#[cfg(test)]
mod test {
    use std::os::getenv;
    use std::path::Path;
    use super::{Feature, FeatureIterator, open};

    fn fixture_path(name: &str) -> Path {
        let envvar = "RUST_GDAL_TEST_FIXTURES";
        let fixtures = match getenv(envvar) {
            Some(p) => Path::new(p),
            None => fail!("Environment variable {} not set", envvar)
        };
        let rv = fixtures.join(name);
        return rv;
    }

    fn assert_almost_eq(a: f64, b: f64) {
        let f: f64 = a / b;
        assert!(f < 1.00001);
        assert!(f > 0.99999);
    }

    #[test]
    fn test_layer_count() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        assert_eq!(ds.layer_count(), 1);
    }

    fn with_features(fixture: &str, f: |FeatureIterator|) {
        let ds = open(&fixture_path(fixture)).unwrap();
        let layer = ds.layer(0).unwrap();
        f(layer.features());
    }

    fn with_first_feature(fixture: &str, f: |Feature|) {
        with_features(fixture, |mut features| f(features.next().unwrap()));
    }

    #[test]
    fn test_iterate_features() {
        with_features("roads.geojson", |mut features| {
            let feature_vec: Vec<Feature> = features.collect();
            assert_eq!(feature_vec.len(), 21);
        });
    }

    #[test]
    fn test_string_field() {
        with_features("roads.geojson", |mut features| {
            let feature = features.next().unwrap();
            assert_eq!(feature.field("highway".to_string())
                              .unwrap()
                              .as_string(),
                       "footway".to_string());
            assert_eq!(
                features.filter(|field| {
                    let highway = field.field("highway".to_string())
                                       .unwrap()
                                       .as_string();
                    highway == "residential".to_string() })
                    .count(),
                2);
        });
    }

    #[test]
    fn test_float_field() {
        with_first_feature("roads.geojson", |feature| {
            assert_almost_eq(
                feature.field("sort_key".to_string())
                       .unwrap()
                       .as_real(),
                -9.0
            );
        });
    }

    #[test]
    fn test_missing_field() {
        with_first_feature("roads.geojson", |feature| {
            assert!(feature.field("no such field".to_string()).is_none());
        });
    }

    #[test]
    fn test_wkt() {
        with_first_feature("roads.geojson", |feature| {
            let wkt = feature.wkt();
            let wkt_ok = format!("{}{}",
                "LINESTRING (26.1019276 44.4302748,",
                "26.1019382 44.4303191,26.1020002 44.4304202)"
                ).to_string();
            assert_eq!(wkt, wkt_ok);
        });
    }

    #[test]
    fn test_json() {
        with_first_feature("roads.geojson", |feature| {
            let json = feature.json();
            let json_ok = format!("{}{}{}{}",
                "{ \"type\": \"LineString\", \"coordinates\": [ ",
                "[ 26.1019276, 44.4302748 ], ",
                "[ 26.1019382, 44.4303191 ], ",
                "[ 26.1020002, 44.4304202 ] ] }"
                ).to_string();
            assert_eq!(json, json_ok);
        });
    }

    #[test]
    fn test_schema() {
        let ds = open(&fixture_path("roads.geojson")).unwrap();
        let layer = ds.layer(0).unwrap();
        let name_list: Vec<String> = layer
            .fields()
            .map(|f| f.name())
            .collect();
        let ok_names = vec!("kind", "sort_key", "is_link", "is_tunnel",
                        "is_bridge", "railway", "highway")
                       .iter().map(|s| s.to_string()).collect();
        assert_eq!(name_list, ok_names);
    }
}
<|endoftext|>


<commit_before>//! Vertex types yielded by the mesh adaptors and their implementations.

use geom;
use geom::graph::node::{self, ApplyTransform};
use math::{BaseFloat, Point2, Point3};
use std::ops::{Deref, DerefMut};

/// A vertex with a specified color.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithColor<V, C> {
    pub vertex: V,
    pub color: C,
}

/// A vertex with some specified texture coordinates.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithTexCoords<V, T = Point2<geom::DefaultScalar>> {
    pub vertex: V,
    pub tex_coords: T,
}

/// A vertex with its normal vector.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithNormal<V, N = geom::vertex::Default> {
    pub vertex: V,
    pub normal: N,
}

// Node Transform application implementations.

impl<S, V, C> ApplyTransform<S> for WithColor<V, C>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, transform: &node::PreparedTransform<S>) -> Self {
        let WithColor { mut vertex, color } = self;
        vertex = vertex.apply_transform(transform);
        WithColor { vertex, color }
    }
}

impl<S, V, T> ApplyTransform<S> for WithTexCoords<V, T>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, transform: &node::PreparedTransform<S>) -> Self {
        let WithTexCoords { mut vertex, tex_coords } = self;
        vertex = vertex.apply_transform(transform);
        WithTexCoords { vertex, tex_coords }
    }
}

impl<S, V, N> ApplyTransform<S> for WithNormal<V, N>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, _transform: &node::PreparedTransform<S>) -> Self {
        //let WithNormal { mut vertex, mut normal } = self;
        //vertex = vertex.apply_transform(transform);
        // TODO: Apply transform to the `normal`.
        unimplemented!();
        //WithNormal { vertex, normal }
    }
}

// Deref implementations for each vertex adaptor to their inner vertex type.

impl<V, C> Deref for WithColor<V, C> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, C> DerefMut for WithColor<V, C> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

impl<V, T> Deref for WithTexCoords<V, T> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, T> DerefMut for WithTexCoords<V, T> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

impl<V, N> Deref for WithNormal<V, N> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, N> DerefMut for WithNormal<V, N> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

// Geometry vertex implementations.

impl<V, C> geom::Vertex for WithColor<V, C>
where
    V: geom::Vertex,
    C: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, T> geom::Vertex for WithTexCoords<V, T>
where
    V: geom::Vertex,
    T: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, N> geom::Vertex for WithNormal<V, N>
where
    V: geom::Vertex,
    N: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, C> geom::Vertex2d for WithColor<V, C>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, T> geom::Vertex2d for WithTexCoords<V, T>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, N> geom::Vertex2d for WithNormal<V, N>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, C> geom::Vertex3d for WithColor<V, C>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}

impl<V, T> geom::Vertex3d for WithTexCoords<V, T>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}

impl<V, N> geom::Vertex3d for WithNormal<V, N>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}
<commit_msg>Add tuple conversion implementations for mesh vertex types<commit_after>//! Vertex types yielded by the mesh adaptors and their implementations.

use geom;
use geom::graph::node::{self, ApplyTransform};
use math::{BaseFloat, Point2, Point3};
use std::ops::{Deref, DerefMut};

/// A vertex with a specified color.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithColor<V, C> {
    pub vertex: V,
    pub color: C,
}

/// A vertex with some specified texture coordinates.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithTexCoords<V, T = Point2<geom::DefaultScalar>> {
    pub vertex: V,
    pub tex_coords: T,
}

/// A vertex with its normal vector.
#[derive(Copy, Clone, Debug, Default, PartialEq)]
pub struct WithNormal<V, N = geom::vertex::Default> {
    pub vertex: V,
    pub normal: N,
}

// Node Transform application implementations.

impl<S, V, C> ApplyTransform<S> for WithColor<V, C>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, transform: &node::PreparedTransform<S>) -> Self {
        let WithColor { mut vertex, color } = self;
        vertex = vertex.apply_transform(transform);
        WithColor { vertex, color }
    }
}

impl<S, V, T> ApplyTransform<S> for WithTexCoords<V, T>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, transform: &node::PreparedTransform<S>) -> Self {
        let WithTexCoords { mut vertex, tex_coords } = self;
        vertex = vertex.apply_transform(transform);
        WithTexCoords { vertex, tex_coords }
    }
}

impl<S, V, N> ApplyTransform<S> for WithNormal<V, N>
where
    V: ApplyTransform<S>,
    S: BaseFloat,
{
    fn apply_transform(self, _transform: &node::PreparedTransform<S>) -> Self {
        //let WithNormal { mut vertex, mut normal } = self;
        //vertex = vertex.apply_transform(transform);
        // TODO: Apply transform to the `normal`.
        unimplemented!();
        //WithNormal { vertex, normal }
    }
}

// Deref implementations for each vertex adaptor to their inner vertex type.

impl<V, C> Deref for WithColor<V, C> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, C> DerefMut for WithColor<V, C> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

impl<V, T> Deref for WithTexCoords<V, T> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, T> DerefMut for WithTexCoords<V, T> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

impl<V, N> Deref for WithNormal<V, N> {
    type Target = V;
    fn deref(&self) -> &Self::Target {
        &self.vertex
    }
}

impl<V, N> DerefMut for WithNormal<V, N> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.vertex
    }
}

// Geometry vertex implementations.

impl<V, C> geom::Vertex for WithColor<V, C>
where
    V: geom::Vertex,
    C: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, T> geom::Vertex for WithTexCoords<V, T>
where
    V: geom::Vertex,
    T: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, N> geom::Vertex for WithNormal<V, N>
where
    V: geom::Vertex,
    N: Clone + Copy + PartialEq,
{
    type Scalar = V::Scalar;
}

impl<V, C> geom::Vertex2d for WithColor<V, C>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, T> geom::Vertex2d for WithTexCoords<V, T>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, N> geom::Vertex2d for WithNormal<V, N>
where
    V: geom::Vertex2d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point2(self) -> Point2<Self::Scalar> {
        self.vertex.point2()
    }
}

impl<V, C> geom::Vertex3d for WithColor<V, C>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}

impl<V, T> geom::Vertex3d for WithTexCoords<V, T>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}

impl<V, N> geom::Vertex3d for WithNormal<V, N>
where
    V: geom::Vertex3d,
    Self: geom::Vertex<Scalar = V::Scalar>,
{
    fn point3(self) -> Point3<Self::Scalar> {
        self.vertex.point3()
    }
}

// For converting from a tuples to vertices.

impl<A, V, C> From<(A, C)> for WithColor<V, C>
where
    A: Into<V>,
{
    fn from((vertex, color): (A, C)) -> Self {
        let vertex = vertex.into();
        WithColor { vertex, color }
    }
}

impl<A, V, T> From<(A, T)> for WithTexCoords<V, T>
where
    A: Into<V>,
{
    fn from((vertex, tex_coords): (A, T)) -> Self {
        let vertex = vertex.into();
        WithTexCoords { vertex, tex_coords }
    }
}

impl<A, V, N> From<(A, N)> for WithNormal<V, N>
where
    A: Into<V>,
{
    fn from((vertex, normal): (A, N)) -> Self {
        let vertex = vertex.into();
        WithNormal { vertex, normal }
    }
}

#[test]
fn test_tuple_conv() {
    use color::named::GREEN;
    let _: Point2<_> = [0.0, 0.0].into();
    let _: WithColor<Point2<_>, _> = ([0.0, 0.0], GREEN).into();
}
<|endoftext|>


<commit_before><commit_msg>Fix serialization test adding event_id field to the JSON string output<commit_after><|endoftext|>