<commit_before>// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
// Licensed under the Apache License, Version 2.0 (see LICENSE).

use std::collections::HashMap;
use std::hash;
use std::sync::atomic;

use cpython::{ObjectProtocol, PyErr, PyType, Python, ToPyObject};
use parking_lot::{Mutex, RwLock};

use crate::externs;
use crate::python::{Fnv, Key, Value};

///
/// A struct that encapsulates interning of python `Value`s as comparable `Key`s.
///
/// To minimize the total amount of time spent in python code comparing objects (represented on
/// the rust side of the FFI boundary as `Value` instances) to one another, this API supports
/// memoizing `Value`s as `Key`s.
///
/// Creating a `Key` involves interning a `Value` under a (private) `InternKey` struct which
/// implements `Hash` and `Eq` using the precomputed python `__hash__` for the `Value` and
/// delegating to python's `__eq__`, respectively.
///
/// Currently `Value`s are interned indefinitely as `Key`s, meaning that they can never
/// be collected: it's possible that this can eventually be improved by either:
///
///   1) switching to directly linking-against or embedding python, such that the `Value`
///      type goes away in favor of direct usage of a python object wrapper struct.
///   2) This structure might begin storing weak-references to `Key`s and/or `Value`s, which
///      would allow the associated `Value` handles to be dropped when they were no longer used.
///      The challenge to this approach is that it would make it more difficult to pass
///      `Key`/`Value` instances across the FFI boundary.
///   3) `Value` could implement `Eq`/`Hash` directly via extern calls to python (although we've
///      avoided doing this so far because it would hide a relatively expensive operation behind
///      those usually-inexpensive traits).
///
/// To avoid deadlocks, methods of Interns require that the GIL is held, and then explicitly release
/// it before acquiring inner locks. That way we can guarantee that these locks are always acquired
/// before the GIL (Value equality in particular might re-acquire it).
///
#[derive(Default)]
pub struct Interns {
  forward_keys: Mutex<HashMap<InternKey, Key, Fnv>>,
  reverse_keys: RwLock<HashMap<Key, Value, Fnv>>,
  id_generator: atomic::AtomicU64,
}

impl Interns {
  pub fn new() -> Interns {
    Interns::default()
  }

  pub fn key_insert(&self, py: Python, v: Value) -> Result<Key, PyErr> {
    let (intern_key, type_id) = {
      let obj = v.to_py_object(py).into();
      (InternKey(v.hash(py)?, obj), (&v.get_type(py)).into())
    };

    py.allow_threads(|| {
      let mut forward_keys = self.forward_keys.lock();
      let key = if let Some(key) = forward_keys.get(&intern_key) {
        *key
      } else {
        let id = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed);
        let key = Key::new(id, type_id);
        self.reverse_keys.write().insert(key, v);
        forward_keys.insert(intern_key, key);
        key
      };
      Ok(key)
    })
  }

  pub fn key_get(&self, k: &Key) -> Value {
    // NB: We do not need to acquire+release the GIL before getting a Value for a Key, because
    // neither `Key::eq` nor `Value::clone` acquire the GIL.
    self.reverse_keys.read().get(k).cloned().unwrap_or_else(|| {
      // N.B.: This panic is effectively an assertion that `Key::new` is only ever called above in
      // `key_insert` under an exclusive lock where it is then inserted in `reverse_keys` before
      // exiting the lock and being returned to the caller. This ensures that all `Key`s in the
      // wild _must_ be in `reverse_keys`. As such, the code involved in generating the panic
      // message should be immaterial and never fire. If it does fire though, then the assertion
      // was proven incorrect and the `Key` is not, in fact, in `reverse_keys`. Since the `Debug`
      // impl for `Key` currently uses this very method to render the `Key` we avoid using the
      // debug formatting for `Key` to avoid generating a panic while panicking.
      panic!(
        "Previously memoized object disappeared for Key {{ id: {}, type_id: {} }}!",
        k.id(),
        k.type_id()
      )
    })
  }
}

struct InternKey(isize, Value);

impl Eq for InternKey {}

impl PartialEq for InternKey {
  fn eq(&self, other: &InternKey) -> bool {
    let gil = Python::acquire_gil();
    externs::equals(gil.python(), &self.1, &other.1)
  }
}

impl hash::Hash for InternKey {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}

struct InternType(isize, PyType);

impl Eq for InternType {}

impl PartialEq for InternType {
  fn eq(&self, other: &InternType) -> bool {
    self.1 == other.1
  }
}

impl hash::Hash for InternType {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}
<commit_msg>[internal] Remove unused `InternType` from `interning.rs` (#13563)<commit_after>// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
// Licensed under the Apache License, Version 2.0 (see LICENSE).

use std::collections::HashMap;
use std::hash;
use std::sync::atomic;

use cpython::{ObjectProtocol, PyErr, Python, ToPyObject};
use parking_lot::{Mutex, RwLock};

use crate::externs;
use crate::python::{Fnv, Key, Value};

///
/// A struct that encapsulates interning of python `Value`s as comparable `Key`s.
///
/// To minimize the total amount of time spent in python code comparing objects (represented on
/// the rust side of the FFI boundary as `Value` instances) to one another, this API supports
/// memoizing `Value`s as `Key`s.
///
/// Creating a `Key` involves interning a `Value` under a (private) `InternKey` struct which
/// implements `Hash` and `Eq` using the precomputed python `__hash__` for the `Value` and
/// delegating to python's `__eq__`, respectively.
///
/// Currently `Value`s are interned indefinitely as `Key`s, meaning that they can never
/// be collected: it's possible that this can eventually be improved by either:
///
///   1) switching to directly linking-against or embedding python, such that the `Value`
///      type goes away in favor of direct usage of a python object wrapper struct.
///   2) This structure might begin storing weak-references to `Key`s and/or `Value`s, which
///      would allow the associated `Value` handles to be dropped when they were no longer used.
///      The challenge to this approach is that it would make it more difficult to pass
///      `Key`/`Value` instances across the FFI boundary.
///   3) `Value` could implement `Eq`/`Hash` directly via extern calls to python (although we've
///      avoided doing this so far because it would hide a relatively expensive operation behind
///      those usually-inexpensive traits).
///
/// To avoid deadlocks, methods of Interns require that the GIL is held, and then explicitly release
/// it before acquiring inner locks. That way we can guarantee that these locks are always acquired
/// before the GIL (Value equality in particular might re-acquire it).
///
#[derive(Default)]
pub struct Interns {
  forward_keys: Mutex<HashMap<InternKey, Key, Fnv>>,
  reverse_keys: RwLock<HashMap<Key, Value, Fnv>>,
  id_generator: atomic::AtomicU64,
}

impl Interns {
  pub fn new() -> Interns {
    Interns::default()
  }

  pub fn key_insert(&self, py: Python, v: Value) -> Result<Key, PyErr> {
    let (intern_key, type_id) = {
      let obj = v.to_py_object(py).into();
      (InternKey(v.hash(py)?, obj), (&v.get_type(py)).into())
    };

    py.allow_threads(|| {
      let mut forward_keys = self.forward_keys.lock();
      let key = if let Some(key) = forward_keys.get(&intern_key) {
        *key
      } else {
        let id = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed);
        let key = Key::new(id, type_id);
        self.reverse_keys.write().insert(key, v);
        forward_keys.insert(intern_key, key);
        key
      };
      Ok(key)
    })
  }

  pub fn key_get(&self, k: &Key) -> Value {
    // NB: We do not need to acquire+release the GIL before getting a Value for a Key, because
    // neither `Key::eq` nor `Value::clone` acquire the GIL.
    self.reverse_keys.read().get(k).cloned().unwrap_or_else(|| {
      // N.B.: This panic is effectively an assertion that `Key::new` is only ever called above in
      // `key_insert` under an exclusive lock where it is then inserted in `reverse_keys` before
      // exiting the lock and being returned to the caller. This ensures that all `Key`s in the
      // wild _must_ be in `reverse_keys`. As such, the code involved in generating the panic
      // message should be immaterial and never fire. If it does fire though, then the assertion
      // was proven incorrect and the `Key` is not, in fact, in `reverse_keys`. Since the `Debug`
      // impl for `Key` currently uses this very method to render the `Key` we avoid using the
      // debug formatting for `Key` to avoid generating a panic while panicking.
      panic!(
        "Previously memoized object disappeared for Key {{ id: {}, type_id: {} }}!",
        k.id(),
        k.type_id()
      )
    })
  }
}

struct InternKey(isize, Value);

impl Eq for InternKey {}

impl PartialEq for InternKey {
  fn eq(&self, other: &InternKey) -> bool {
    let gil = Python::acquire_gil();
    externs::equals(gil.python(), &self.1, &other.1)
  }
}

impl hash::Hash for InternKey {
  fn hash<H: hash::Hasher>(&self, state: &mut H) {
    self.0.hash(state);
  }
}
<|endoftext|>


<commit_before>use serialization::Serializable;
use abomonation::{Abomonation, encode, decode};
// use columnar::Columnar;

impl<T: Abomonation+Clone> Serializable for T {
    fn encode(typed: Self, bytes: &mut Vec<u8>) {
        encode(&typed, bytes);
    }
    fn decode(bytes: &mut [u8]) -> Result<Self, &mut [u8]> {
        let result = if let Ok(result) = decode::<T>(bytes) { Ok(result.clone()) } else { Err(()) };
        if let Ok(result) = result { Ok(result) } else { Err (bytes) }
    }
}
<commit_msg>abomo 0.2.0 API update<commit_after>use serialization::Serializable;
use abomonation::{Abomonation, encode, decode};
// use columnar::Columnar;

impl<T: Abomonation+Clone> Serializable for T {
    fn encode(typed: Self, bytes: &mut Vec<u8>) {
        encode(&typed, bytes);
    }
    fn decode(bytes: &mut [u8]) -> Result<Self, &mut [u8]> {
        let result = if let Ok((result,_)) = decode::<T>(bytes) { Ok(result.clone()) } else { Err(()) };
        if let Ok(result) = result { Ok(result) } else { Err (bytes) }
    }
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Interfaces to the operating system provided random number
//! generators.

pub use self::imp::OsRng;

#[cfg(unix, not(target_os = "ios"))]
mod imp {
    use io::{IoResult, File};
    use path::Path;
    use rand::Rng;
    use rand::reader::ReaderRng;
    use result::{Ok, Err};

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    /// - iOS: calls SecRandomCopyBytes as /dev/(u)random is sandboxed
    /// This does not block.
    #[cfg(unix)]
    pub struct OsRng {
        inner: ReaderRng<File>
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            let reader = try!(File::open(&Path::new("/dev/urandom")));
            let reader_rng = ReaderRng::new(reader);

            Ok(OsRng { inner: reader_rng })
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            self.inner.next_u32()
        }
        fn next_u64(&mut self) -> u64 {
            self.inner.next_u64()
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            self.inner.fill_bytes(v)
        }
    }
}

#[cfg(target_os = "ios")]
mod imp {
    extern crate libc;

    use collections::Collection;
    use io::{IoResult};
    use kinds::marker;
    use mem;
    use os;
    use rand::Rng;
    use result::{Ok};
    use self::libc::{c_int, size_t};
    use slice::MutableSlice;

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    /// - iOS: calls SecRandomCopyBytes as /dev/(u)random is sandboxed
    /// This does not block.
    pub struct OsRng {
        marker: marker::NoCopy
    }

    struct SecRandom;

    static kSecRandomDefault: *const SecRandom = 0 as *const SecRandom;

    #[link(name = "Security", kind = "framework")]
    extern "C" {
        fn SecRandomCopyBytes(rnd: *const SecRandom,
                              count: size_t, bytes: *mut u8) -> c_int;
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            Ok(OsRng {marker: marker::NoCopy} )
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            let mut v = [0u8, .. 4];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn next_u64(&mut self) -> u64 {
            let mut v = [0u8, .. 8];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            let ret = unsafe {
                SecRandomCopyBytes(kSecRandomDefault, v.len() as size_t, v.as_mut_ptr())
            };
            if ret == -1 {
                fail!("couldn't generate random bytes: {}", os::last_os_error());
            }
        }
    }
}

#[cfg(windows)]
mod imp {
    extern crate libc;

    use core_collections::Collection;
    use io::{IoResult, IoError};
    use mem;
    use ops::Drop;
    use os;
    use rand::Rng;
    use result::{Ok, Err};
    use rt::stack;
    use self::libc::{DWORD, BYTE, LPCSTR, BOOL};
    use self::libc::types::os::arch::extra::{LONG_PTR};
    use slice::MutableSlice;

    type HCRYPTPROV = LONG_PTR;

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    ///
    /// This does not block.
    pub struct OsRng {
        hcryptprov: HCRYPTPROV
    }

    static PROV_RSA_FULL: DWORD = 1;
    static CRYPT_SILENT: DWORD = 64;
    static CRYPT_VERIFYCONTEXT: DWORD = 0xF0000000;
    static NTE_BAD_SIGNATURE: DWORD = 0x80090006;

    #[allow(non_snake_case_functions)]
    extern "system" {
        fn CryptAcquireContextA(phProv: *mut HCRYPTPROV,
                                pszContainer: LPCSTR,
                                pszProvider: LPCSTR,
                                dwProvType: DWORD,
                                dwFlags: DWORD) -> BOOL;
        fn CryptGenRandom(hProv: HCRYPTPROV,
                          dwLen: DWORD,
                          pbBuffer: *mut BYTE) -> BOOL;
        fn CryptReleaseContext(hProv: HCRYPTPROV, dwFlags: DWORD) -> BOOL;
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            let mut hcp = 0;
            let mut ret = unsafe {
                CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
                                     PROV_RSA_FULL,
                                     CRYPT_VERIFYCONTEXT | CRYPT_SILENT)
            };

            // FIXME #13259:
            // It turns out that if we can't acquire a context with the
            // NTE_BAD_SIGNATURE error code, the documentation states:
            //
            //     The provider DLL signature could not be verified. Either the
            //     DLL or the digital signature has been tampered with.
            //
            // Sounds fishy, no? As it turns out, our signature can be bad
            // because our Thread Information Block (TIB) isn't exactly what it
            // expects. As to why, I have no idea. The only data we store in the
            // TIB is the stack limit for each thread, but apparently that's
            // enough to make the signature valid.
            //
            // Furthermore, this error only happens the *first* time we call
            // CryptAcquireContext, so we don't have to worry about future
            // calls.
            //
            // Anyway, the fix employed here is that if we see this error, we
            // pray that we're not close to the end of the stack, temporarily
            // set the stack limit to 0 (what the TIB originally was), acquire a
            // context, and then reset the stack limit.
            //
            // Again, I'm not sure why this is the fix, nor why we're getting
            // this error. All I can say is that this seems to allow libnative
            // to progress where it otherwise would be hindered. Who knew?
            if ret == 0 && os::errno() as DWORD == NTE_BAD_SIGNATURE {
                unsafe {
                    let limit = stack::get_sp_limit();
                    stack::record_sp_limit(0);
                    ret = CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
                                               PROV_RSA_FULL,
                                               CRYPT_VERIFYCONTEXT | CRYPT_SILENT);
                    stack::record_sp_limit(limit);
                }
            }

            if ret == 0 {
                Err(IoError::last_error())
            } else {
                Ok(OsRng { hcryptprov: hcp })
            }
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            let mut v = [0u8, .. 4];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn next_u64(&mut self) -> u64 {
            let mut v = [0u8, .. 8];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            let ret = unsafe {
                CryptGenRandom(self.hcryptprov, v.len() as DWORD,
                               v.as_mut_ptr())
            };
            if ret == 0 {
                fail!("couldn't generate random bytes: {}", os::last_os_error());
            }
        }
    }

    impl Drop for OsRng {
        fn drop(&mut self) {
            let ret = unsafe {
                CryptReleaseContext(self.hcryptprov, 0)
            };
            if ret == 0 {
                fail!("couldn't release context: {}", os::last_os_error());
            }
        }
    }
}

#[cfg(test)]
mod test {
    use prelude::*;

    use super::OsRng;
    use rand::Rng;
    use task;

    #[test]
    fn test_os_rng() {
        let mut r = OsRng::new().unwrap();

        r.next_u32();
        r.next_u64();

        let mut v = [0u8, .. 1000];
        r.fill_bytes(v);
    }

    #[test]
    fn test_os_rng_tasks() {

        let mut txs = vec!();
        for _ in range(0u, 20) {
            let (tx, rx) = channel();
            txs.push(tx);
            task::spawn(proc() {
                // wait until all the tasks are ready to go.
                rx.recv();

                // deschedule to attempt to interleave things as much
                // as possible (XXX: is this a good test?)
                let mut r = OsRng::new().unwrap();
                task::deschedule();
                let mut v = [0u8, .. 1000];

                for _ in range(0u, 100) {
                    r.next_u32();
                    task::deschedule();
                    r.next_u64();
                    task::deschedule();
                    r.fill_bytes(v);
                    task::deschedule();
                }
            })
        }

        // start all the tasks
        for tx in txs.iter() {
            tx.send(())
        }
    }
}
<commit_msg>auto merge of #16742 : vhbit/rust/ios-ffi-fix, r=alexcrichton<commit_after>// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Interfaces to the operating system provided random number
//! generators.

pub use self::imp::OsRng;

#[cfg(unix, not(target_os = "ios"))]
mod imp {
    use io::{IoResult, File};
    use path::Path;
    use rand::Rng;
    use rand::reader::ReaderRng;
    use result::{Ok, Err};

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    /// - iOS: calls SecRandomCopyBytes as /dev/(u)random is sandboxed
    /// This does not block.
    #[cfg(unix)]
    pub struct OsRng {
        inner: ReaderRng<File>
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            let reader = try!(File::open(&Path::new("/dev/urandom")));
            let reader_rng = ReaderRng::new(reader);

            Ok(OsRng { inner: reader_rng })
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            self.inner.next_u32()
        }
        fn next_u64(&mut self) -> u64 {
            self.inner.next_u64()
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            self.inner.fill_bytes(v)
        }
    }
}

#[cfg(target_os = "ios")]
mod imp {
    extern crate libc;

    use collections::Collection;
    use io::{IoResult};
    use kinds::marker;
    use mem;
    use os;
    use rand::Rng;
    use result::{Ok};
    use self::libc::{c_int, size_t};
    use slice::MutableSlice;

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    /// - iOS: calls SecRandomCopyBytes as /dev/(u)random is sandboxed
    /// This does not block.
    pub struct OsRng {
        marker: marker::NoCopy
    }

    #[repr(C)]
    struct SecRandom;

    static kSecRandomDefault: *const SecRandom = 0 as *const SecRandom;

    #[link(name = "Security", kind = "framework")]
    extern "C" {
        fn SecRandomCopyBytes(rnd: *const SecRandom,
                              count: size_t, bytes: *mut u8) -> c_int;
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            Ok(OsRng {marker: marker::NoCopy} )
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            let mut v = [0u8, .. 4];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn next_u64(&mut self) -> u64 {
            let mut v = [0u8, .. 8];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            let ret = unsafe {
                SecRandomCopyBytes(kSecRandomDefault, v.len() as size_t, v.as_mut_ptr())
            };
            if ret == -1 {
                fail!("couldn't generate random bytes: {}", os::last_os_error());
            }
        }
    }
}

#[cfg(windows)]
mod imp {
    extern crate libc;

    use core_collections::Collection;
    use io::{IoResult, IoError};
    use mem;
    use ops::Drop;
    use os;
    use rand::Rng;
    use result::{Ok, Err};
    use rt::stack;
    use self::libc::{DWORD, BYTE, LPCSTR, BOOL};
    use self::libc::types::os::arch::extra::{LONG_PTR};
    use slice::MutableSlice;

    type HCRYPTPROV = LONG_PTR;

    /// A random number generator that retrieves randomness straight from
    /// the operating system. Platform sources:
    ///
    /// - Unix-like systems (Linux, Android, Mac OSX): read directly from
    ///   `/dev/urandom`.
    /// - Windows: calls `CryptGenRandom`, using the default cryptographic
    ///   service provider with the `PROV_RSA_FULL` type.
    ///
    /// This does not block.
    pub struct OsRng {
        hcryptprov: HCRYPTPROV
    }

    static PROV_RSA_FULL: DWORD = 1;
    static CRYPT_SILENT: DWORD = 64;
    static CRYPT_VERIFYCONTEXT: DWORD = 0xF0000000;
    static NTE_BAD_SIGNATURE: DWORD = 0x80090006;

    #[allow(non_snake_case_functions)]
    extern "system" {
        fn CryptAcquireContextA(phProv: *mut HCRYPTPROV,
                                pszContainer: LPCSTR,
                                pszProvider: LPCSTR,
                                dwProvType: DWORD,
                                dwFlags: DWORD) -> BOOL;
        fn CryptGenRandom(hProv: HCRYPTPROV,
                          dwLen: DWORD,
                          pbBuffer: *mut BYTE) -> BOOL;
        fn CryptReleaseContext(hProv: HCRYPTPROV, dwFlags: DWORD) -> BOOL;
    }

    impl OsRng {
        /// Create a new `OsRng`.
        pub fn new() -> IoResult<OsRng> {
            let mut hcp = 0;
            let mut ret = unsafe {
                CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
                                     PROV_RSA_FULL,
                                     CRYPT_VERIFYCONTEXT | CRYPT_SILENT)
            };

            // FIXME #13259:
            // It turns out that if we can't acquire a context with the
            // NTE_BAD_SIGNATURE error code, the documentation states:
            //
            //     The provider DLL signature could not be verified. Either the
            //     DLL or the digital signature has been tampered with.
            //
            // Sounds fishy, no? As it turns out, our signature can be bad
            // because our Thread Information Block (TIB) isn't exactly what it
            // expects. As to why, I have no idea. The only data we store in the
            // TIB is the stack limit for each thread, but apparently that's
            // enough to make the signature valid.
            //
            // Furthermore, this error only happens the *first* time we call
            // CryptAcquireContext, so we don't have to worry about future
            // calls.
            //
            // Anyway, the fix employed here is that if we see this error, we
            // pray that we're not close to the end of the stack, temporarily
            // set the stack limit to 0 (what the TIB originally was), acquire a
            // context, and then reset the stack limit.
            //
            // Again, I'm not sure why this is the fix, nor why we're getting
            // this error. All I can say is that this seems to allow libnative
            // to progress where it otherwise would be hindered. Who knew?
            if ret == 0 && os::errno() as DWORD == NTE_BAD_SIGNATURE {
                unsafe {
                    let limit = stack::get_sp_limit();
                    stack::record_sp_limit(0);
                    ret = CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
                                               PROV_RSA_FULL,
                                               CRYPT_VERIFYCONTEXT | CRYPT_SILENT);
                    stack::record_sp_limit(limit);
                }
            }

            if ret == 0 {
                Err(IoError::last_error())
            } else {
                Ok(OsRng { hcryptprov: hcp })
            }
        }
    }

    impl Rng for OsRng {
        fn next_u32(&mut self) -> u32 {
            let mut v = [0u8, .. 4];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn next_u64(&mut self) -> u64 {
            let mut v = [0u8, .. 8];
            self.fill_bytes(v);
            unsafe { mem::transmute(v) }
        }
        fn fill_bytes(&mut self, v: &mut [u8]) {
            let ret = unsafe {
                CryptGenRandom(self.hcryptprov, v.len() as DWORD,
                               v.as_mut_ptr())
            };
            if ret == 0 {
                fail!("couldn't generate random bytes: {}", os::last_os_error());
            }
        }
    }

    impl Drop for OsRng {
        fn drop(&mut self) {
            let ret = unsafe {
                CryptReleaseContext(self.hcryptprov, 0)
            };
            if ret == 0 {
                fail!("couldn't release context: {}", os::last_os_error());
            }
        }
    }
}

#[cfg(test)]
mod test {
    use prelude::*;

    use super::OsRng;
    use rand::Rng;
    use task;

    #[test]
    fn test_os_rng() {
        let mut r = OsRng::new().unwrap();

        r.next_u32();
        r.next_u64();

        let mut v = [0u8, .. 1000];
        r.fill_bytes(v);
    }

    #[test]
    fn test_os_rng_tasks() {

        let mut txs = vec!();
        for _ in range(0u, 20) {
            let (tx, rx) = channel();
            txs.push(tx);
            task::spawn(proc() {
                // wait until all the tasks are ready to go.
                rx.recv();

                // deschedule to attempt to interleave things as much
                // as possible (XXX: is this a good test?)
                let mut r = OsRng::new().unwrap();
                task::deschedule();
                let mut v = [0u8, .. 1000];

                for _ in range(0u, 100) {
                    r.next_u32();
                    task::deschedule();
                    r.next_u64();
                    task::deschedule();
                    r.fill_bytes(v);
                    task::deschedule();
                }
            })
        }

        // start all the tasks
        for tx in txs.iter() {
            tx.send(())
        }
    }
}
<|endoftext|>


<commit_before>// Copyright Dan Schatzberg, 2015. This file is part of Genesis.

// Genesis is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Genesis is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.

// You should have received a copy of the GNU Affero General Public License
// along with Genesis.  If not, see <http://www.gnu.org/licenses/>.
use super::serial;
use core::mem::transmute;
use core::slice;
use fixedvec::FixedVec;
use memory::{Frame, FrameAllocator, FrameRange, PAddr};
use memory::first_fit_allocator::FirstFitAllocator;
use multiboot::{self, MemoryType, Multiboot};
use spin;

unsafe fn early_paddr_to_slice<'a>(p: multiboot::PAddr,
                                   sz: usize)
                                   -> Option<&'a [u8]> {
    if (sz < (1 << 30)) && ((p + sz as u64) < (1 << 30)) {
        let ptr = transmute(p + 0xFFFFFFFFC0000000);
        Some(slice::from_raw_parts(ptr, sz))
    } else {
        None
    }
}

#[derive(Clone, Copy, Debug)]
struct MemoryRegion {
    start: PAddr,
    end: PAddr,
}

impl MemoryRegion {
    const fn new(start: PAddr, end: PAddr) -> MemoryRegion {
        MemoryRegion {
            start: start,
            end: end,
        }
    }

    fn trim_below(&mut self, addr: PAddr) {
        if self.start < addr && self.end > addr {
            self.start = addr;
        }
    }

    fn trim_above(&mut self, addr: PAddr) {
        if self.start < addr && self.end > addr {
            self.end = addr;
        }
    }
}

type RegionVec = FixedVec<'static, MemoryRegion>;

lazy_static! {
    static ref REGIONS: spin::RwLock<RegionVec> = {
        const REGIONS_SIZE: usize = 256;
        static mut REGIONS_MEM: [MemoryRegion; REGIONS_SIZE] =
            [MemoryRegion::new(PAddr::from_u64(0), PAddr::from_u64(0));
             REGIONS_SIZE];
        // Unsafe to take a mutable reference of a static.
        // We instantly store it behind a Rwlock, so this is safe
        unsafe {
            spin::RwLock::new(FixedVec::new(&mut REGIONS_MEM))
        }
    };
}

/// Initial Rust entry point.
#[no_mangle]
pub extern "C" fn arch_init(multiboot_addr: PAddr) -> ! {
    unsafe {
        serial::init();
    }
    debug!("Serial Initialized");
    debug!("Multiboot Structure loaded at {:#X}", multiboot_addr);
    let mb = unsafe {
                 Multiboot::new(multiboot_addr.as_u64(), early_paddr_to_slice)
             }
             .expect("Could not access a Multiboot structure");

    {
        let mut regions = REGIONS.write();
        discover_memory(&mb, &mut *regions);
    }
    let regions = REGIONS.read();
    debug!("Available Memory:");
    for region in regions.iter() {
        debug!("{:?}", region);
    }

    let allocator = FirstFitAllocator::get();
    populate_allocator(&*regions, allocator);
    map_available_memory(&*regions, allocator);
    loop {}
}

/// Report kernel physical memory range (not including boot code/data)
fn kernel_memory_range() -> (PAddr, PAddr) {
    // kbegin and kend are defined as symbols in the linker script
    extern "C" {
        static kbegin: u8;
        static kend: u8;
    }

    const MASK: u64 = (1 << 12) - 1;
    let kbegin_addr = {
        let ptr: *const _ = &kbegin;
        ptr as u64 & !MASK
    };
    let kend_addr = {
        let ptr: *const _ = &kend;
        (ptr as u64 + MASK) & !MASK
    };
    (PAddr::from_u64(kbegin_addr), PAddr::from_u64(kend_addr))
}

/// Discover available memory from the Multiboot structure
fn discover_memory(mb: &Multiboot, regions: &mut RegionVec) {
    let (kbegin, kend) = kernel_memory_range();
    debug!("kbegin = {:#X}, kend = {:#X}", kbegin, kend);
    let mem_regions = mb.memory_regions()
                        .expect("Could not find Multiboot memory map");
    for region in mem_regions {
        let start = PAddr::from_u64(region.base_address());
        let end = PAddr::from_u64(region.base_address() + region.length());
        let mem_type = match region.memory_type() {
            MemoryType::RAM => "RAM",
            MemoryType::Unusable => "Unusable",
        };
        info!("{:#17X} - {:#17X}: {}", start, end, mem_type);
        if region.memory_type() == MemoryType::RAM {
            let mut reg = MemoryRegion::new(start, end);
            reg.trim_below(kend);
            reg.trim_above(kbegin);
            if reg.start != reg.end {
                if let Err(e) = regions.push(reg) {
                    warn!("Could not store usable region {:#?}: {:?}",
                          region,
                          e)
                }
            }
        }
    }
}

/// Populate the memory allocator with accessible frames
fn populate_allocator<Allocator: FrameAllocator>(regions: &RegionVec,
                                                 allocator: &Allocator) {
    const INITIAL_MAP: PAddr = PAddr::from_u64(1 << 30);
    let accessible_frames = regions.iter().filter_map(|reg| {
        let start_frame = Frame::up(reg.start);
        let end_frame = Frame::down(if reg.end < INITIAL_MAP {
            reg.end
        } else {
            INITIAL_MAP
        });
        let range = FrameRange::new(start_frame, end_frame);
        if start_frame.start_address() >= INITIAL_MAP || range.nframes() == 0 {
            None
        } else {
            Some(range)
        }
    });
    for range in accessible_frames {
        unsafe { allocator.free_range_manual(range) };
    }
}

fn map_available_memory<Allocator: FrameAllocator>(regions: &RegionVec,
                                                   allocator: &Allocator) {

}
<commit_msg>Improve debug print<commit_after>// Copyright Dan Schatzberg, 2015. This file is part of Genesis.

// Genesis is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Genesis is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.

// You should have received a copy of the GNU Affero General Public License
// along with Genesis.  If not, see <http://www.gnu.org/licenses/>.
use super::serial;
use core::mem::transmute;
use core::slice;
use fixedvec::FixedVec;
use memory::{Frame, FrameAllocator, FrameRange, PAddr};
use memory::first_fit_allocator::FirstFitAllocator;
use multiboot::{self, MemoryType, Multiboot};
use spin;

unsafe fn early_paddr_to_slice<'a>(p: multiboot::PAddr,
                                   sz: usize)
                                   -> Option<&'a [u8]> {
    if (sz < (1 << 30)) && ((p + sz as u64) < (1 << 30)) {
        let ptr = transmute(p + 0xFFFFFFFFC0000000);
        Some(slice::from_raw_parts(ptr, sz))
    } else {
        None
    }
}

#[derive(Clone, Copy, Debug)]
struct MemoryRegion {
    start: PAddr,
    end: PAddr,
}

impl MemoryRegion {
    const fn new(start: PAddr, end: PAddr) -> MemoryRegion {
        MemoryRegion {
            start: start,
            end: end,
        }
    }

    fn trim_below(&mut self, addr: PAddr) {
        if self.start < addr && self.end > addr {
            self.start = addr;
        }
    }

    fn trim_above(&mut self, addr: PAddr) {
        if self.start < addr && self.end > addr {
            self.end = addr;
        }
    }
}

type RegionVec = FixedVec<'static, MemoryRegion>;

lazy_static! {
    static ref REGIONS: spin::RwLock<RegionVec> = {
        const REGIONS_SIZE: usize = 256;
        static mut REGIONS_MEM: [MemoryRegion; REGIONS_SIZE] =
            [MemoryRegion::new(PAddr::from_u64(0), PAddr::from_u64(0));
             REGIONS_SIZE];
        // Unsafe to take a mutable reference of a static.
        // We instantly store it behind a Rwlock, so this is safe
        unsafe {
            spin::RwLock::new(FixedVec::new(&mut REGIONS_MEM))
        }
    };
}

/// Initial Rust entry point.
#[no_mangle]
pub extern "C" fn arch_init(multiboot_addr: PAddr) -> ! {
    unsafe {
        serial::init();
    }
    debug!("Serial Initialized");
    debug!("Multiboot Structure loaded at {:#X}", multiboot_addr);
    let mb = unsafe {
                 Multiboot::new(multiboot_addr.as_u64(), early_paddr_to_slice)
             }
             .expect("Could not access a Multiboot structure");

    {
        let mut regions = REGIONS.write();
        discover_memory(&mb, &mut *regions);
    }
    let regions = REGIONS.read();
    debug!("Available Memory:");
    for region in regions.iter() {
        debug!("{:#17X} - {:#17X}", region.start, region.end);
    }

    let allocator = FirstFitAllocator::get();
    populate_allocator(&*regions, allocator);
    map_available_memory(&*regions, allocator);
    loop {}
}

/// Report kernel physical memory range (not including boot code/data)
fn kernel_memory_range() -> (PAddr, PAddr) {
    // kbegin and kend are defined as symbols in the linker script
    extern "C" {
        static kbegin: u8;
        static kend: u8;
    }

    const MASK: u64 = (1 << 12) - 1;
    let kbegin_addr = {
        let ptr: *const _ = &kbegin;
        ptr as u64 & !MASK
    };
    let kend_addr = {
        let ptr: *const _ = &kend;
        (ptr as u64 + MASK) & !MASK
    };
    (PAddr::from_u64(kbegin_addr), PAddr::from_u64(kend_addr))
}

/// Discover available memory from the Multiboot structure
fn discover_memory(mb: &Multiboot, regions: &mut RegionVec) {
    let (kbegin, kend) = kernel_memory_range();
    debug!("kbegin = {:#X}, kend = {:#X}", kbegin, kend);
    let mem_regions = mb.memory_regions()
                        .expect("Could not find Multiboot memory map");
    for region in mem_regions {
        let start = PAddr::from_u64(region.base_address());
        let end = PAddr::from_u64(region.base_address() + region.length());
        let mem_type = match region.memory_type() {
            MemoryType::RAM => "RAM",
            MemoryType::Unusable => "Unusable",
        };
        info!("{:#17X} - {:#17X}: {}", start, end, mem_type);
        if region.memory_type() == MemoryType::RAM {
            let mut reg = MemoryRegion::new(start, end);
            reg.trim_below(kend);
            reg.trim_above(kbegin);
            if reg.start != reg.end {
                if let Err(e) = regions.push(reg) {
                    warn!("Could not store usable region {:#?}: {:?}",
                          region,
                          e)
                }
            }
        }
    }
}

/// Populate the memory allocator with accessible frames
fn populate_allocator<Allocator: FrameAllocator>(regions: &RegionVec,
                                                 allocator: &Allocator) {
    const INITIAL_MAP: PAddr = PAddr::from_u64(1 << 30);
    let accessible_frames = regions.iter().filter_map(|reg| {
        let start_frame = Frame::up(reg.start);
        let end_frame = Frame::down(if reg.end < INITIAL_MAP {
            reg.end
        } else {
            INITIAL_MAP
        });
        let range = FrameRange::new(start_frame, end_frame);
        if start_frame.start_address() >= INITIAL_MAP || range.nframes() == 0 {
            None
        } else {
            Some(range)
        }
    });
    for range in accessible_frames {
        unsafe { allocator.free_range_manual(range) };
    }
}

fn map_available_memory<Allocator: FrameAllocator>(regions: &RegionVec,
                                                   allocator: &Allocator) {

}
<|endoftext|>


<commit_before>use std::iter::Peekable;

use ::opcodes::{AddressingMode, OpCode};
use assembler::token::Token;

#[derive(Debug, PartialEq)]
pub struct ParserError {
    message: String,
}

impl ParserError {
    fn expected_instruction(line: u32) -> ParserError {
        ParserError::from(format!("Instruction expected. Line {}", line))
    }

    fn invalid_opcode_addressing_mode_combination(line: u32) -> ParserError {
        ParserError::from(format!("Invalid addressing mode for opcode. Line {}", line))
    }

    fn unexpected_eol(line: u32) -> ParserError {
        ParserError::from(format!("Unexpected end of line. Line {}", line))
    }
}

impl From<String> for ParserError {
    fn from(error: String) -> ParserError {
        ParserError { message: error }
    }
}

impl<'a> From<&'a str> for ParserError {
    fn from(error: &str) -> ParserError {
        ParserError { message: error.into() }
    }
}

pub struct Parser {
    tokens: Vec<Vec<Token>>,
    line: u32,
}

/// Parser processes a list of 6502 Assembly tokens
impl Parser {
    pub fn new(tokens: Vec<Vec<Token>>) -> Parser {
        Parser {
            tokens: tokens,
            line: 0,
        }
    }

    /// Processes its tokens and either returns them to the caller
    /// or produces an error
    pub fn parse(&mut self) -> Result<Vec<Vec<Token>>, ParserError> {
        let successful_result = self.tokens.iter().map(|v| v.clone()).collect();

        for line in &self.tokens {
            self.line += 1;
            let mut peeker = line.iter().peekable();
            // Check what starts the line
            let token = peeker.peek().unwrap().clone();

            match *token {
                Token::Label(_) => {
                    // if its a label, consume it and move on
                    peeker.next();
                    if let None = peeker.peek() {
                        // If its just a label, thats fine
                        return Ok(successful_result);
                    }
                    let next = *peeker.peek().unwrap();
                    if let &Token::OpCode(ref mnemonic) = next {
                        peeker.next();
                        Self::handle_opcode(&mut peeker, mnemonic, self.line)?;
                    } else {
                        return Err(ParserError::expected_instruction(self.line));
                    }
                }
                Token::OpCode(ref mnemonic) => {
                    peeker.next();
                    Self::handle_opcode(&mut peeker, mnemonic, self.line)?;
                }
                _ => (),
            }
        }

        Ok(successful_result)
    }

    fn handle_opcode<'a, I>(mut peeker: &mut Peekable<I>,
                            mnemonic: &str,
                            line: u32)
                            -> Result<(), ParserError>
        where I: Iterator<Item = &'a Token>
    {
        // Determine an addressing mode that was attempted,
        let addressing_mode = if let None = peeker.peek() {
            // if there was no token, then its implied
            AddressingMode::Implied
        } else {
            // Otherwise, lets try and convert
            // the next token to a mode
            let next = *peeker.peek().unwrap();
            next.to_addressing_mode()
        };

        if let Some(opcode) = OpCode::from_mnemonic_and_addressing_mode(mnemonic.clone(),
                                                                        addressing_mode) {
            Ok(())
        } else {
            Err(ParserError::invalid_opcode_addressing_mode_combination(line))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ::assembler::token::{ImmediateBase, Token};

    #[test]
    fn errors_on_multiple_labels() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::Label("METHOD".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::Immediate("10".into(),
                                                                ImmediateBase::Base16)]]);

        assert_eq!(Err(ParserError::expected_instruction(1)), parser.parse());
    }

    #[test]
    fn does_not_error_on_single_label() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::Immediate("10".into(),
                                                                ImmediateBase::Base16)]]);

        assert_eq!(&[Token::Label("MAIN".into()),
                     Token::OpCode("LDA".into()),
                     Token::Immediate("10".into(), ImmediateBase::Base16)],
                   &parser.parse().unwrap()[0][..]);
    }

    #[test]
    fn can_detect_invalid_addressing_modes() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDX".into()),
                                               Token::IndirectY("10".into())]]);

        assert_eq!(Err(ParserError::invalid_opcode_addressing_mode_combination(1)),
                   parser.parse());
    }

    #[test]
    fn does_not_error_on_valid_addressing_modes() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::IndirectY("10".into())]]);

        let result = parser.parse().unwrap();
        assert_eq!(&[Token::Label("MAIN".into()),
                     Token::OpCode("LDA".into()),
                     Token::IndirectY("10".into())],
                   &result[0][..]);
    }

    #[test]
    fn does_not_error_on_label_only_line() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into())],
                                          vec![Token::OpCode("LDA".into()),
                                               Token::Absolute("4400".into())]]);

        let result = parser.parse().unwrap();

        assert_eq!(&[Token::Label("MAIN".into())], &result[0][..]);
        assert_eq!(&[Token::OpCode("LDA".into()), Token::Absolute("4400".into())],
                   &result[1][..]);
    }

    #[test]
    fn does_not_error_on_implied_addressing_mode() {
        let mut parser = Parser::new(vec![vec![Token::OpCode("NOP".into())],
                                          vec![Token::OpCode("BRK".into())],
                                          vec![Token::OpCode("CLC".into())]]);

        let result = parser.parse().unwrap();

        assert_eq!(&[Token::OpCode("NOP".into())], &result[0][..]);
        assert_eq!(&[Token::OpCode("BRK".into())], &result[1][..]);
        assert_eq!(&[Token::OpCode("CLC".into())], &result[2][..]);
    }

    #[test]
    fn does_error_on_implied_addressing_mode_opcodes_that_have_arguments() {
        let mut parser = Parser::new(vec![vec![Token::OpCode("NOP".into()),
                                               Token::Absolute("4400".into())]]);

        assert_eq!(Err(ParserError::invalid_opcode_addressing_mode_combination(1)),
                   parser.parse());
    }
}<commit_msg>Stop multiple instructions on one line<commit_after>use std::iter::Peekable;

use ::opcodes::{AddressingMode, OpCode};
use assembler::token::Token;

#[derive(Debug, PartialEq)]
pub struct ParserError {
    message: String,
}

impl ParserError {
    fn expected_instruction(line: u32) -> ParserError {
        ParserError::from(format!("Instruction expected. Line {}", line))
    }

    fn invalid_opcode_addressing_mode_combination(line: u32) -> ParserError {
        ParserError::from(format!("Invalid addressing mode for opcode. Line {}", line))
    }

    fn unexpected_eol(line: u32) -> ParserError {
        ParserError::from(format!("Unexpected end of line. Line {}", line))
    }

    fn expected_eol(line: u32) -> ParserError {
        ParserError::from(format!("Expected end of line. Line {}", line))
    }
}

impl From<String> for ParserError {
    fn from(error: String) -> ParserError {
        ParserError { message: error }
    }
}

impl<'a> From<&'a str> for ParserError {
    fn from(error: &str) -> ParserError {
        ParserError { message: error.into() }
    }
}

pub struct Parser {
    tokens: Vec<Vec<Token>>,
    line: u32,
}

/// Parser processes a list of 6502 Assembly tokens
impl Parser {
    pub fn new(tokens: Vec<Vec<Token>>) -> Parser {
        Parser {
            tokens: tokens,
            line: 0,
        }
    }

    /// Processes its tokens and either returns them to the caller
    /// or produces an error
    pub fn parse(&mut self) -> Result<Vec<Vec<Token>>, ParserError> {
        let successful_result = self.tokens.iter().map(|v| v.clone()).collect();

        for line in &self.tokens {
            self.line += 1;
            let mut peeker = line.iter().peekable();
            // Check what starts the line
            let token = peeker.peek().unwrap().clone();

            match *token {
                Token::Label(_) => {
                    // if its a label, consume it and move on
                    peeker.next();
                    if let None = peeker.peek() {
                        // If its just a label, thats fine
                        return Ok(successful_result);
                    }
                    let next = *peeker.peek().unwrap();
                    if let &Token::OpCode(ref mnemonic) = next {
                        peeker.next();
                        Self::handle_opcode(&mut peeker, mnemonic, self.line)?;
                    } else {
                        return Err(ParserError::expected_instruction(self.line));
                    }
                }
                Token::OpCode(ref mnemonic) => {
                    peeker.next();
                    Self::handle_opcode(&mut peeker, mnemonic, self.line)?;
                }
                _ => (),
            }
        }

        Ok(successful_result)
    }

    fn handle_opcode<'a, I>(mut peeker: &mut Peekable<I>,
                            mnemonic: &str,
                            line: u32)
                            -> Result<(), ParserError>
        where I: Iterator<Item = &'a Token>
    {
        // Determine an addressing mode that was attempted,
        let addressing_mode = if let None = peeker.peek() {
            // if there was no token, then its implied
            AddressingMode::Implied
        } else {
            // Otherwise, lets try and convert
            // the next token to a mode
            let next = *peeker.peek().unwrap();
            // Skip the argument
            peeker.next();
            next.to_addressing_mode()
        };

        if let Some(opcode) = OpCode::from_mnemonic_and_addressing_mode(mnemonic.clone(),
                                                                        addressing_mode) {
            // There should be nothing else now:
            if let None = peeker.peek() {
                Ok(())
            } else {
                Err(ParserError::expected_eol(line))
            }
        } else {
            Err(ParserError::invalid_opcode_addressing_mode_combination(line))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ::assembler::token::{ImmediateBase, Token};

    #[test]
    fn errors_on_multiple_labels() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::Label("METHOD".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::Immediate("10".into(),
                                                                ImmediateBase::Base16)]]);

        assert_eq!(Err(ParserError::expected_instruction(1)), parser.parse());
    }

    #[test]
    fn does_not_error_on_single_label() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::Immediate("10".into(),
                                                                ImmediateBase::Base16)]]);

        assert_eq!(&[Token::Label("MAIN".into()),
                     Token::OpCode("LDA".into()),
                     Token::Immediate("10".into(), ImmediateBase::Base16)],
                   &parser.parse().unwrap()[0][..]);
    }

    #[test]
    fn can_detect_invalid_addressing_modes() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDX".into()),
                                               Token::IndirectY("10".into())]]);

        assert_eq!(Err(ParserError::invalid_opcode_addressing_mode_combination(1)),
                   parser.parse());
    }

    #[test]
    fn does_not_error_on_valid_addressing_modes() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::IndirectY("10".into())]]);

        let result = parser.parse().unwrap();
        assert_eq!(&[Token::Label("MAIN".into()),
                     Token::OpCode("LDA".into()),
                     Token::IndirectY("10".into())],
                   &result[0][..]);
    }

    #[test]
    fn does_not_error_on_label_only_line() {
        let mut parser = Parser::new(vec![vec![Token::Label("MAIN".into())],
                                          vec![Token::OpCode("LDA".into()),
                                               Token::Absolute("4400".into())]]);

        let result = parser.parse().unwrap();

        assert_eq!(&[Token::Label("MAIN".into())], &result[0][..]);
        assert_eq!(&[Token::OpCode("LDA".into()), Token::Absolute("4400".into())],
                   &result[1][..]);
    }

    #[test]
    fn does_not_error_on_implied_addressing_mode() {
        let mut parser = Parser::new(vec![vec![Token::OpCode("NOP".into())],
                                          vec![Token::OpCode("BRK".into())],
                                          vec![Token::OpCode("CLC".into())]]);

        let result = parser.parse().unwrap();

        assert_eq!(&[Token::OpCode("NOP".into())], &result[0][..]);
        assert_eq!(&[Token::OpCode("BRK".into())], &result[1][..]);
        assert_eq!(&[Token::OpCode("CLC".into())], &result[2][..]);
    }

    #[test]
    fn does_error_on_implied_addressing_mode_opcodes_that_have_arguments() {
        let mut parser = Parser::new(vec![vec![Token::OpCode("NOP".into()),
                                               Token::Absolute("4400".into())]]);

        assert_eq!(Err(ParserError::invalid_opcode_addressing_mode_combination(1)),
                   parser.parse());
    }

    #[test]
    fn does_error_when_multiple_instructions_are_not_split_across_lines() {
        let mut parser = Parser::new(vec![vec![Token::OpCode("LDA".into()),
                                               Token::Absolute("4400".into()),
                                               Token::OpCode("LDA".into()),
                                               Token::Absolute("4400".into()),
                                            ]]);

        assert_eq!(Err(ParserError::expected_eol(1)), parser.parse());
    }
}<|endoftext|>


<commit_before>use rusqlite::Connection;
use std::result;

#[derive(Debug)]
pub struct JobRequireVariablePair {
    pub id: i32,
    pub fk_job: i32,
    pub fk_variable_pair: i32,
}

pub fn table_create_job_require_variable_pair(conn: &Connection) {
    let load_table = conn.execute("CREATE TABLE JOB_REQUIRE_VALUE (
                  id            INTEGER PRIMARY KEY ASC,
                  job           INTEGER NOT NULL,
                  variable_pair INTEGER NOT NULL,
                  FOREIGN KEY(job) REFERENCES JOB(id) ON UPDATE CASCADE
                  FOREIGN KEY(variable_pair) REFERENCES VARIABLE_PAIR(id) ON UPDATE CASCADE
                  )", &[]);
    if load_table.is_err() {
        println!("table_create_job Failed {:?}", load_table);
        return;
    }
    load_table.unwrap();
}

pub fn insert_job_require_variable_pair(conn: &Connection, job :&i32, variable_pair: &i32) -> Result<i32, &'static str> {
    let me = JobRequireVariablePair {
        id: 0,
        fk_job: *job,
        fk_variable_pair: *variable_pair,

    };
    let variable_pair_instance = conn.execute("INSERT INTO JOB_REQUIRE_VALUE (job, variable_pair)
                  VALUES (?1, ?2)",
                 &[&me.fk_job, &me.fk_variable_pair]);
    if variable_pair_instance.is_err() {
        return Err("Insert failed");
    }
    variable_pair_instance.unwrap();
    return Ok(0);
}

pub fn list_job_require_variable_pair(conn: &Connection)-> Vec<JobRequireVariablePair> {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair  FROM VARIABLE_PAIR").unwrap();
    let wraped_fs_file_iter = stmt.query_map(&[], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    });
    let mut items = Vec::<JobRequireVariablePair>::new();
    if wraped_fs_file_iter.is_err() {
        return items;

    }
    let fs_file_iter = wraped_fs_file_iter.unwrap();
    for person in fs_file_iter {

        items.push(person.unwrap());
    }
    return items;
}

pub fn job_require_variable_pair_list(conn: &Connection) {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair FROM VARIABLE_PAIR").unwrap();
    let person_iter = stmt.query_map(&[], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    }).unwrap();

    for person in person_iter {
        println!("Found variable_pair {:?}", person.unwrap());
    }
}

pub fn pk_job_require_variable_pair_by_all(conn: &Connection, job: &i32, variable_pair: &i32, pk: &mut i32) -> Result<i32, &'static str>{
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair  FROM VARIABLE_PAIR WHERE name = ?1").unwrap();
    let variable_pair_iter = stmt.query_map(&[job, variable_pair], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    });
    if variable_pair_iter.is_err() {
        return Err("Insert failed dfdf");
    }
    let result = variable_pair_iter.unwrap();
    let mut found = 0;
    let mut items = Vec::<i32>::new();
    for person in result {
        let bill= person.unwrap();
        *pk = bill.id;
        found = 1;
    }
    if found != 0 {
        return Ok(found);
    }
    return Err("None found");
}
<commit_msg>Return result as a result<commit_after>use rusqlite::Connection;
use std::result;

#[derive(Debug)]
pub struct JobRequireVariablePair {
    pub id: i32,
    pub fk_job: i32,
    pub fk_variable_pair: i32,
}

pub fn table_create_job_require_variable_pair(conn: &Connection) {
    let load_table = conn.execute("CREATE TABLE JOB_REQUIRE_VALUE (
                  id            INTEGER PRIMARY KEY ASC,
                  job           INTEGER NOT NULL,
                  variable_pair INTEGER NOT NULL,
                  FOREIGN KEY(job) REFERENCES JOB(id) ON UPDATE CASCADE
                  FOREIGN KEY(variable_pair) REFERENCES VARIABLE_PAIR(id) ON UPDATE CASCADE
                  )", &[]);
    if load_table.is_err() {
        println!("table_create_job Failed {:?}", load_table);
        return;
    }
    load_table.unwrap();
}

pub fn insert_job_require_variable_pair(conn: &Connection, job :&i32, variable_pair: &i32) -> Result<i32, &'static str> {
    let me = JobRequireVariablePair {
        id: 0,
        fk_job: *job,
        fk_variable_pair: *variable_pair,

    };
    let variable_pair_instance = conn.execute("INSERT INTO JOB_REQUIRE_VALUE (job, variable_pair)
                  VALUES (?1, ?2)",
                 &[&me.fk_job, &me.fk_variable_pair]);
    if variable_pair_instance.is_err() {
        return Err("Insert failed");
    }
    variable_pair_instance.unwrap();
    return Ok(0);
}

pub fn list_job_require_variable_pair(conn: &Connection)-> Vec<JobRequireVariablePair> {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair  FROM VARIABLE_PAIR").unwrap();
    let wraped_fs_file_iter = stmt.query_map(&[], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    });
    let mut items = Vec::<JobRequireVariablePair>::new();
    if wraped_fs_file_iter.is_err() {
        return items;

    }
    let fs_file_iter = wraped_fs_file_iter.unwrap();
    for person in fs_file_iter {

        items.push(person.unwrap());
    }
    return items;
}

pub fn job_require_variable_pair_list(conn: &Connection) {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair FROM VARIABLE_PAIR").unwrap();
    let person_iter = stmt.query_map(&[], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    }).unwrap();

    for person in person_iter {
        println!("Found variable_pair {:?}", person.unwrap());
    }
}

pub fn pk_job_require_variable_pair_by_all(conn: &Connection, job: &i32, variable_pair: &i32) -> Result<i32, &'static str>{
    let mut output = 0;
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable_pair  FROM VARIABLE_PAIR WHERE name = ?1").unwrap();
    let variable_pair_iter = stmt.query_map(&[job, variable_pair], |row| {
        JobRequireVariablePair {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable_pair: row.get(2),
        }
    });
    if variable_pair_iter.is_err() {
        return Err("Insert failed dfdf");
    }
    let result = variable_pair_iter.unwrap();
    let mut found = 0;
    let mut items = Vec::<i32>::new();
    for person in result {
        let bill= person.unwrap();
        output = bill.id;
        found = 1;
    }
    if found != 0 {
        return Ok(output);
    }
    return Err("None found");
}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
extern crate env_logger;
extern crate argparse;
extern crate quire;
extern crate lithos;
extern crate time;
extern crate scan_dir;
extern crate rustc_serialize;

use std::env;
use std::io::{BufReader, BufRead};
use std::fs::{File, remove_dir_all};
use std::path::{PathBuf, Path};
use std::process::exit;
use std::collections::HashSet;
use std::collections::BTreeMap;
use std::collections::VecDeque;

use time::{Tm, Duration, now_utc};
use quire::parse_config;
use argparse::{ArgumentParser, Parse, ParseOption, StoreTrue, StoreConst};
use argparse::{Print};
use rustc_serialize::json;

use lithos::master_config::MasterConfig;
use lithos::tree_config::TreeConfig;
use lithos::child_config::ChildConfig;

#[derive(Clone, Copy, Debug)]
enum Action {
    Used,
    Unused,
    DeleteUnused,
}

fn main() {

    if env::var("RUST_LOG").is_err() {
        env::set_var("RUST_LOG", "warn");
    }
    env_logger::init().unwrap();
    let mut config_file = PathBuf::from("/etc/lithos/master.yaml");
    let mut verbose = false;
    let mut ver_min = 0;
    let mut ver_max = 1000;
    let mut action = Action::Used;
    let mut days = None::<u32>;
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("Show used/unused images and clean if needed");
        ap.refer(&mut config_file)
          .add_option(&["-C", "--config"], Parse,
            "Name of the global configuration file \
             (default /etc/lithos/master.yaml)")
          .metavar("FILE");
        ap.refer(&mut days)
          .add_option(&["-D", "--history-days"], ParseOption,
            r"Keep images that used no more than DAYS ago.
              There is no reasonable default, so you should specify this
              argument or --versions-min to get sane behavior.")
          .metavar("DAYS");
        ap.refer(&mut ver_min)
          .add_option(&["--vmin", "--versions-min"], Parse,
            r"Keep minimum NUM versions (even if they are older than DAYS).
              Default is 0 which means keep only current version.")
          .metavar("NUM");
        ap.refer(&mut ver_max)
          .add_option(&["--vmax", "--versions-max"], Parse,
            r"Keep maximum NUM versions
              (even if need to delete images more recent than DAYS).
              Default is 1000.")
          .metavar("NUM");
        ap.refer(&mut verbose)
          .add_option(&["-v", "--verbose"], StoreTrue,
            "Verbose output");
        ap.refer(&mut action)
          .add_option(&["--used"], StoreConst(Action::Used),
            "Show used images")
          .add_option(&["--unused"], StoreConst(Action::Unused),
            "Show unused images")
          .add_option(&["--delete-unused"], StoreConst(Action::DeleteUnused),
            "Delete unused images");
        ap.add_option(&["--version"],
            Print(env!("CARGO_PKG_VERSION").to_string()),
            "Show version of the lithos");
        ap.parse_args_or_exit();
    }
    let master: MasterConfig = match parse_config(&config_file,
        &MasterConfig::validator(), Default::default()) {
        Ok(cfg) => cfg,
        Err(e) => {
            error!("Can't parse config: {}", e);
            exit(1);
        }
    };
    let tm = days.map(|days| now_utc() - Duration::days(days as i64));
    let (used, dirs) = match find_used_images(&master, &config_file,
        tm, ver_min, ver_max)
    {
        Ok((used, dirs)) => (used, dirs),
        Err(e) => {
            error!("Error finding out used images: {}", e);
            exit(1);
        }
    };
    match action {
        Action::Used => {
            for i in used {
                println!("{:?}", i);
            }
        }
        Action::Unused => {
            let unused = find_unused(&used, &dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                println!("{:?}", i);
            }
        }
        Action::DeleteUnused => {
            let unused = find_unused(&used, &dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                if verbose {
                    println!("Deleting {:?}", i);
                }
                remove_dir_all(&i)
                    .map_err(|e| error!("Error removing {:?}: {}", i, e)).ok();
            }
        }
    }
}

fn find_unused(used: &HashSet<PathBuf>, dirs: &HashSet<PathBuf>)
    -> Result<Vec<PathBuf>, scan_dir::Error>
{
    let mut unused = Vec::new();
    for dir in dirs.iter() {
        try!(scan_dir::ScanDir::dirs().read(dir, |iter| {
            for (entry, _) in iter {
                let path = entry.path().to_path_buf();
                if !used.contains(&path) {
                    unused.push(path);
                }
            }
        }));
    }
    Ok(unused)
}

fn find_used_images(master: &MasterConfig, master_file: &Path,
    min_time: Option<Tm>, ver_min: u32, ver_max: u32)
    -> Result<(HashSet<PathBuf>, HashSet<PathBuf>), String>
{
    let config_dir = master_file.parent().unwrap().join(&master.sandboxes_dir);
    let mut images = HashSet::new();
    let mut image_dirs = HashSet::new();
    let childval = ChildConfig::mapping_validator();
    try!(try!(scan_dir::ScanDir::files().read(&config_dir, |iter| {
        let yamls = iter.filter(|&(_, ref name)| name.ends_with(".yaml"));
        for (entry, tree_fname) in yamls {
            let tree_name = &tree_fname[..tree_fname.len()-5];  // strip .yaml
            let tree_config: TreeConfig = try!(parse_config(&entry.path(),
                &TreeConfig::validator(), Default::default()));
            image_dirs.insert(tree_config.image_dir.clone());

            let cfg = master_file.parent().unwrap()
                .join(&master.processes_dir)
                .join(tree_config.config_file.as_ref().unwrap_or(
                    &PathBuf::from(&(tree_name.to_string() + ".yaml"))));
            let all_children: BTreeMap<String, ChildConfig>;
            all_children = try!(parse_config(&cfg, &childval, Default::default())
                .map_err(|e| format!("Can't read child config {:?}: {}",
                                     tree_config.config_file, e)));
            for child in all_children.values() {
                // Current are always added
                images.insert(tree_config.image_dir.join(&child.image));
            }

            let logname = master.config_log_dir
                .join(format!("{}.log", tree_name));
            // TODO(tailhook) look in log rotations
            let log = try!(File::open(&logname)
                .map_err(|e| format!("Can't read log file {:?}: {}", logname, e)));
            let mut configs;
            configs = VecDeque::<(Tm, BTreeMap<String, ChildConfig>)>::new();
            for (line_no, line) in BufReader::new(log).lines().enumerate() {
                let line = try!(line
                    .map_err(|e| format!("Readline error: {}", e)));
                let mut iter = line.splitn(2, " ");
                let (tm, cfg) = match (iter.next(), iter.next()) {
                        (Some(""), None) => continue, // last line, probably
                        (Some(date), Some(config)) => {
                            // TODO(tailhook) remove or check tree name
                            (try!(time::strptime(date, "%Y-%m-%dT%H:%M:%SZ")
                                 .map_err(|_| format!("Bad time at {:?}:{}",
                                    logname, line_no))),
                             try!(json::decode(config)
                                 .map_err(|_| format!("Bad config at {:?}:{}",
                                    logname, line_no))),
                            )
                        }
                        _ => {
                            return Err(format!("Bad line at {:?}:{}",
                                               logname, line_no));
                        }
                    };
                if let Some(&(_, ref ocfg)) = configs.back(){
                    if *ocfg == cfg {
                        continue;
                    }
                }
                configs.push_back((tm, cfg));
                if configs.len() >= ver_max as usize {
                    configs.pop_front();
                }
            }
            min_time.map(|min_time| {
                while configs.len() > ver_min as usize {
                    if let Some(&(tm, _)) = configs.front() {
                        if tm > min_time {
                            break;
                        }
                    } else {
                        break;
                    }
                    configs.pop_front();
                }
            });
            for &(_, ref cfg) in configs.iter() {
                for child in cfg.values() {
                    // Current are always added
                    images.insert(tree_config.image_dir.join(&child.image));
                }
            }
        }
        Ok(())
    }).map_err(|e| format!("Read dir error: {}", e))));
    Ok((images, image_dirs))
}
<commit_msg>Temporary fix for skipping symlinks<commit_after>#[macro_use] extern crate log;
extern crate env_logger;
extern crate argparse;
extern crate quire;
extern crate lithos;
extern crate time;
extern crate scan_dir;
extern crate rustc_serialize;

use std::env;
use std::io::{BufReader, BufRead};
use std::fs::{File, remove_dir_all};
use std::path::{PathBuf, Path};
use std::process::exit;
use std::collections::HashSet;
use std::collections::BTreeMap;
use std::collections::VecDeque;

use time::{Tm, Duration, now_utc};
use quire::parse_config;
use argparse::{ArgumentParser, Parse, ParseOption, StoreTrue, StoreConst};
use argparse::{Print};
use rustc_serialize::json;

use lithos::master_config::MasterConfig;
use lithos::tree_config::TreeConfig;
use lithos::child_config::ChildConfig;

#[derive(Clone, Copy, Debug)]
enum Action {
    Used,
    Unused,
    DeleteUnused,
}

fn main() {

    if env::var("RUST_LOG").is_err() {
        env::set_var("RUST_LOG", "warn");
    }
    env_logger::init().unwrap();
    let mut config_file = PathBuf::from("/etc/lithos/master.yaml");
    let mut verbose = false;
    let mut ver_min = 0;
    let mut ver_max = 1000;
    let mut action = Action::Used;
    let mut days = None::<u32>;
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("Show used/unused images and clean if needed");
        ap.refer(&mut config_file)
          .add_option(&["-C", "--config"], Parse,
            "Name of the global configuration file \
             (default /etc/lithos/master.yaml)")
          .metavar("FILE");
        ap.refer(&mut days)
          .add_option(&["-D", "--history-days"], ParseOption,
            r"Keep images that used no more than DAYS ago.
              There is no reasonable default, so you should specify this
              argument or --versions-min to get sane behavior.")
          .metavar("DAYS");
        ap.refer(&mut ver_min)
          .add_option(&["--vmin", "--versions-min"], Parse,
            r"Keep minimum NUM versions (even if they are older than DAYS).
              Default is 0 which means keep only current version.")
          .metavar("NUM");
        ap.refer(&mut ver_max)
          .add_option(&["--vmax", "--versions-max"], Parse,
            r"Keep maximum NUM versions
              (even if need to delete images more recent than DAYS).
              Default is 1000.")
          .metavar("NUM");
        ap.refer(&mut verbose)
          .add_option(&["-v", "--verbose"], StoreTrue,
            "Verbose output");
        ap.refer(&mut action)
          .add_option(&["--used"], StoreConst(Action::Used),
            "Show used images")
          .add_option(&["--unused"], StoreConst(Action::Unused),
            "Show unused images")
          .add_option(&["--delete-unused"], StoreConst(Action::DeleteUnused),
            "Delete unused images");
        ap.add_option(&["--version"],
            Print(env!("CARGO_PKG_VERSION").to_string()),
            "Show version of the lithos");
        ap.parse_args_or_exit();
    }
    let master: MasterConfig = match parse_config(&config_file,
        &MasterConfig::validator(), Default::default()) {
        Ok(cfg) => cfg,
        Err(e) => {
            error!("Can't parse config: {}", e);
            exit(1);
        }
    };
    let tm = days.map(|days| now_utc() - Duration::days(days as i64));
    let (used, dirs) = match find_used_images(&master, &config_file,
        tm, ver_min, ver_max)
    {
        Ok((used, dirs)) => (used, dirs),
        Err(e) => {
            error!("Error finding out used images: {}", e);
            exit(1);
        }
    };
    match action {
        Action::Used => {
            for i in used {
                println!("{:?}", i);
            }
        }
        Action::Unused => {
            let unused = find_unused(&used, &dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                println!("{:?}", i);
            }
        }
        Action::DeleteUnused => {
            let unused = find_unused(&used, &dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                if verbose {
                    println!("Deleting {:?}", i);
                }
                remove_dir_all(&i)
                    .map_err(|e| error!("Error removing {:?}: {}", i, e)).ok();
            }
        }
    }
}

fn find_unused(used: &HashSet<PathBuf>, dirs: &HashSet<PathBuf>)
    -> Result<Vec<PathBuf>, scan_dir::Error>
{
    let mut unused = Vec::new();
    for dir in dirs.iter() {
        try!(scan_dir::ScanDir::dirs().read(dir, |iter| {
            for (entry, _) in iter {
                // we know that file_type works, because scan_dir already
                // tried
                let typ = entry.file_type().unwrap();

                if typ.is_symlink() {
                    // Temporary fix: symlinks are ignored
                    // TODO(tailhook) symlinks are used images (optionally)
                    continue;
                }
                let path = entry.path().to_path_buf();
                if !used.contains(&path) {
                    unused.push(path);
                }
            }
        }));
    }
    Ok(unused)
}

fn find_used_images(master: &MasterConfig, master_file: &Path,
    min_time: Option<Tm>, ver_min: u32, ver_max: u32)
    -> Result<(HashSet<PathBuf>, HashSet<PathBuf>), String>
{
    let config_dir = master_file.parent().unwrap().join(&master.sandboxes_dir);
    let mut images = HashSet::new();
    let mut image_dirs = HashSet::new();
    let childval = ChildConfig::mapping_validator();
    try!(try!(scan_dir::ScanDir::files().read(&config_dir, |iter| {
        let yamls = iter.filter(|&(_, ref name)| name.ends_with(".yaml"));
        for (entry, tree_fname) in yamls {
            let tree_name = &tree_fname[..tree_fname.len()-5];  // strip .yaml
            let tree_config: TreeConfig = try!(parse_config(&entry.path(),
                &TreeConfig::validator(), Default::default()));
            image_dirs.insert(tree_config.image_dir.clone());

            let cfg = master_file.parent().unwrap()
                .join(&master.processes_dir)
                .join(tree_config.config_file.as_ref().unwrap_or(
                    &PathBuf::from(&(tree_name.to_string() + ".yaml"))));
            let all_children: BTreeMap<String, ChildConfig>;
            all_children = try!(parse_config(&cfg, &childval, Default::default())
                .map_err(|e| format!("Can't read child config {:?}: {}",
                                     tree_config.config_file, e)));
            for child in all_children.values() {
                // Current are always added
                images.insert(tree_config.image_dir.join(&child.image));
            }

            let logname = master.config_log_dir
                .join(format!("{}.log", tree_name));
            // TODO(tailhook) look in log rotations
            let log = try!(File::open(&logname)
                .map_err(|e| format!("Can't read log file {:?}: {}", logname, e)));
            let mut configs;
            configs = VecDeque::<(Tm, BTreeMap<String, ChildConfig>)>::new();
            for (line_no, line) in BufReader::new(log).lines().enumerate() {
                let line = try!(line
                    .map_err(|e| format!("Readline error: {}", e)));
                let mut iter = line.splitn(2, " ");
                let (tm, cfg) = match (iter.next(), iter.next()) {
                        (Some(""), None) => continue, // last line, probably
                        (Some(date), Some(config)) => {
                            // TODO(tailhook) remove or check tree name
                            (try!(time::strptime(date, "%Y-%m-%dT%H:%M:%SZ")
                                 .map_err(|_| format!("Bad time at {:?}:{}",
                                    logname, line_no))),
                             try!(json::decode(config)
                                 .map_err(|_| format!("Bad config at {:?}:{}",
                                    logname, line_no))),
                            )
                        }
                        _ => {
                            return Err(format!("Bad line at {:?}:{}",
                                               logname, line_no));
                        }
                    };
                if let Some(&(_, ref ocfg)) = configs.back(){
                    if *ocfg == cfg {
                        continue;
                    }
                }
                configs.push_back((tm, cfg));
                if configs.len() >= ver_max as usize {
                    configs.pop_front();
                }
            }
            min_time.map(|min_time| {
                while configs.len() > ver_min as usize {
                    if let Some(&(tm, _)) = configs.front() {
                        if tm > min_time {
                            break;
                        }
                    } else {
                        break;
                    }
                    configs.pop_front();
                }
            });
            for &(_, ref cfg) in configs.iter() {
                for child in cfg.values() {
                    // Current are always added
                    images.insert(tree_config.image_dir.join(&child.image));
                }
            }
        }
        Ok(())
    }).map_err(|e| format!("Read dir error: {}", e))));
    Ok((images, image_dirs))
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::io::Write;
use std::fs::File;
use std::path;
use std::fs;

use bindgen::config::{Config, Language};
use bindgen::ir::{Item, Function};
use bindgen::monomorph::TemplateSpecialization;
use bindgen::writer::{ListType, Source, SourceWriter};

pub struct Bindings {
    config: Config,
    items: Vec<Item>,
    functions: Vec<Function>,
    template_specializations: Vec<TemplateSpecialization>,
}

impl Bindings {
    pub fn new(config: Config,
               items: Vec<Item>,
               functions: Vec<Function>,
               template_specializations: Vec<TemplateSpecialization>) -> Bindings {
        Bindings {
            config: config,
            items: items,
            functions: functions,
            template_specializations: template_specializations,
        }
    }

    pub fn write_to_file(&self, path: &str) {
        if let Some(parent) = path::Path::new(path).parent() {
            fs::create_dir_all(parent).unwrap();
        }

        self.write(File::create(path).unwrap());
    }

    pub fn write<F: Write>(&self, file: F) {
        let mut out = SourceWriter::new(file, &self.config);

        if let Some(ref f) = self.config.header {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
        if let Some(ref f) = self.config.include_guard {
            out.new_line_if_not_start();
            out.write(&format!("#ifndef {}", f));
            out.new_line();
            out.write(&format!("#define {}", f));
            out.new_line();
        }
        if self.config.include_version {
            out.new_line_if_not_start();
            out.write(&format!("/* Generated with cbindgen:{} */",
                      ::bindgen::config::VERSION));
            out.new_line();
        }
        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }

        out.new_line_if_not_start();
        if self.config.language == Language::C {
            out.write("#include <stdint.h>");
            out.new_line();
            out.write("#include <stdlib.h>");
            out.new_line();
            out.write("#include <stdbool.h>");
        } else {
            out.write("#include <cstdint>");
            out.new_line();
            out.write("#include <cstdlib>");
        }
        out.new_line();

        if self.config.language == Language::Cxx {
            out.new_line_if_not_start();
            out.write("extern \"C\" {");
            out.new_line();

            let mut wrote_namespace: bool = false;
            if let Some(ref namespace) = self.config.namespace {
                wrote_namespace = true;

                out.new_line();
                out.write("namespace ");
                out.write(namespace);
                out.write(" {");
            }
            if let Some(ref namespaces) = self.config.namespaces {
                wrote_namespace = true;
                for namespace in namespaces {
                    out.new_line();
                    out.write("namespace ");
                    out.write(namespace);
                    out.write(" {");
                }
            }
            if wrote_namespace {
                out.new_line();
            }
        }

        for item in &self.items {
            out.new_line_if_not_start();
            match item {
                &Item::Enum(ref x) => x.write(&self.config, &mut out),
                &Item::Struct(ref x) => x.write(&self.config, &mut out),
                &Item::OpaqueItem(ref x) => x.write(&self.config, &mut out),
                &Item::Typedef(ref x) => x.write(&self.config, &mut out),
                &Item::Specialization(_) => {
                    unreachable!("should not encounter a specialization in a generated library")
                }
            }
            out.new_line();
        }

        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }

        for function in &self.functions {
            out.new_line_if_not_start();
            function.write(&self.config, &mut out);
            out.new_line();
        }

        if self.config.language == Language::Cxx {
            let mut wrote_namespace: bool = false;
            if let Some(ref namespaces) = self.config.namespaces {
                wrote_namespace = true;

                for namespace in namespaces.iter().rev() {
                    out.new_line_if_not_start();
                    out.write("} // namespace ");
                    out.write(namespace);
                }
            }
            if let Some(ref namespace) = self.config.namespace {
                wrote_namespace = true;

                out.new_line_if_not_start();
                out.write("} // namespace ");
                out.write(namespace);
            }
            if wrote_namespace {
                out.new_line();
            }

            out.new_line_if_not_start();
            out.write("} // extern \"C\"");
            out.new_line();
        }

        if self.config.structure.generic_template_specialization &&
           self.config.language == Language::Cxx {
          for template in &self.template_specializations {
            out.new_line_if_not_start();
            out.write("template<");
            for (i, param) in template.generic.generic_params.iter().enumerate() {
                if i != 0 {
                    out.write(", ")
                }
                out.write("typename ");
                out.write(param);
            }
            out.write(">");
            out.new_line();
            out.write(&format!("struct {};", template.generic.name));
            out.new_line();

            for &(ref monomorph_path, ref generic_values) in &template.monomorphs {
              out.new_line();
              out.write("template<>");
              out.new_line();
              out.write(&format!("struct {}<", template.generic.name));
              out.write_horizontal_source_list(generic_values, ListType::Join(", "));
              out.write(&format!("> : public {}", monomorph_path));
              out.open_brace();
              out.close_brace(true);
              out.new_line();
            }
          }
        }

        if self.config.language == Language::Cxx {
            out.new_line_if_not_start();
            out.write("static_assert(sizeof(float) == 4);");
            out.new_line();
            out.write("static_assert(sizeof(double) == 8);");
            out.new_line();
        }

        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
        if let Some(ref f) = self.config.include_guard {
            out.new_line_if_not_start();
            out.write(&format!("#endif // {}", f));
            out.new_line();
        }
        if let Some(ref f) = self.config.trailer {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
    }
}
<commit_msg>Wrap template specialization's in namespaces<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::io::Write;
use std::fs::File;
use std::path;
use std::fs;

use bindgen::config::{Config, Language};
use bindgen::ir::{Item, Function};
use bindgen::monomorph::TemplateSpecialization;
use bindgen::writer::{ListType, Source, SourceWriter};

pub struct Bindings {
    config: Config,
    items: Vec<Item>,
    functions: Vec<Function>,
    template_specializations: Vec<TemplateSpecialization>,
}

impl Bindings {
    pub fn new(config: Config,
               items: Vec<Item>,
               functions: Vec<Function>,
               template_specializations: Vec<TemplateSpecialization>) -> Bindings {
        Bindings {
            config: config,
            items: items,
            functions: functions,
            template_specializations: template_specializations,
        }
    }

    pub fn write_to_file(&self, path: &str) {
        if let Some(parent) = path::Path::new(path).parent() {
            fs::create_dir_all(parent).unwrap();
        }

        self.write(File::create(path).unwrap());
    }

    pub fn write<F: Write>(&self, file: F) {
        let mut out = SourceWriter::new(file, &self.config);

        if let Some(ref f) = self.config.header {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
        if let Some(ref f) = self.config.include_guard {
            out.new_line_if_not_start();
            out.write(&format!("#ifndef {}", f));
            out.new_line();
            out.write(&format!("#define {}", f));
            out.new_line();
        }
        if self.config.include_version {
            out.new_line_if_not_start();
            out.write(&format!("/* Generated with cbindgen:{} */",
                      ::bindgen::config::VERSION));
            out.new_line();
        }
        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }

        out.new_line_if_not_start();
        if self.config.language == Language::C {
            out.write("#include <stdint.h>");
            out.new_line();
            out.write("#include <stdlib.h>");
            out.new_line();
            out.write("#include <stdbool.h>");
        } else {
            out.write("#include <cstdint>");
            out.new_line();
            out.write("#include <cstdlib>");
        }
        out.new_line();

        if self.config.language == Language::Cxx {
            out.new_line_if_not_start();
            out.write("extern \"C\" {");
            out.new_line();

            self.open_namespaces(&mut out);
        }

        for item in &self.items {
            out.new_line_if_not_start();
            match item {
                &Item::Enum(ref x) => x.write(&self.config, &mut out),
                &Item::Struct(ref x) => x.write(&self.config, &mut out),
                &Item::OpaqueItem(ref x) => x.write(&self.config, &mut out),
                &Item::Typedef(ref x) => x.write(&self.config, &mut out),
                &Item::Specialization(_) => {
                    unreachable!("should not encounter a specialization in a generated library")
                }
            }
            out.new_line();
        }

        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }

        for function in &self.functions {
            out.new_line_if_not_start();
            function.write(&self.config, &mut out);
            out.new_line();
        }

        if self.config.language == Language::Cxx {
            self.close_namespaces(&mut out);

            out.new_line_if_not_start();
            out.write("} // extern \"C\"");
            out.new_line();
        }

        if self.config.structure.generic_template_specialization &&
           self.config.language == Language::Cxx {
            self.open_namespaces(&mut out);
            for template in &self.template_specializations {
              out.new_line_if_not_start();
              out.write("template<");
              for (i, param) in template.generic.generic_params.iter().enumerate() {
                  if i != 0 {
                      out.write(", ")
                  }
                  out.write("typename ");
                  out.write(param);
              }
              out.write(">");
              out.new_line();
              out.write(&format!("struct {};", template.generic.name));
              out.new_line();

              for &(ref monomorph_path, ref generic_values) in &template.monomorphs {
                out.new_line();
                out.write("template<>");
                out.new_line();
                out.write(&format!("struct {}<", template.generic.name));
                out.write_horizontal_source_list(generic_values, ListType::Join(", "));
                out.write(&format!("> : public {}", monomorph_path));
                out.open_brace();
                out.close_brace(true);
                out.new_line();
              }
            }
            self.close_namespaces(&mut out);
        }

        if self.config.language == Language::Cxx {
            out.new_line_if_not_start();
            out.write("static_assert(sizeof(float) == 4);");
            out.new_line();
            out.write("static_assert(sizeof(double) == 8);");
            out.new_line();
        }

        if let Some(ref f) = self.config.autogen_warning {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
        if let Some(ref f) = self.config.include_guard {
            out.new_line_if_not_start();
            out.write(&format!("#endif // {}", f));
            out.new_line();
        }
        if let Some(ref f) = self.config.trailer {
            out.new_line_if_not_start();
            out.write(&f);
            out.new_line();
        }
    }

    pub fn open_namespaces<F: Write>(&self, out: &mut SourceWriter<F>) {
        let mut wrote_namespace: bool = false;
        if let Some(ref namespace) = self.config.namespace {
            wrote_namespace = true;

            out.new_line();
            out.write("namespace ");
            out.write(namespace);
            out.write(" {");
        }
        if let Some(ref namespaces) = self.config.namespaces {
            wrote_namespace = true;
            for namespace in namespaces {
                out.new_line();
                out.write("namespace ");
                out.write(namespace);
                out.write(" {");
            }
        }
        if wrote_namespace {
            out.new_line();
        }
    }

    pub fn close_namespaces<F: Write>(&self, out: &mut SourceWriter<F>) {
        let mut wrote_namespace: bool = false;
        if let Some(ref namespaces) = self.config.namespaces {
            wrote_namespace = true;

            for namespace in namespaces.iter().rev() {
                out.new_line_if_not_start();
                out.write("} // namespace ");
                out.write(namespace);
            }
        }
        if let Some(ref namespace) = self.config.namespace {
            wrote_namespace = true;

            out.new_line_if_not_start();
            out.write("} // namespace ");
            out.write(namespace);
        }
        if wrote_namespace {
            out.new_line();
        }
    }
}
<|endoftext|>