<commit_before>use rustc::hir::def_id::DefId;
use rustc::traits::{self, Reveal, SelectionContext};
use rustc::ty::subst::Substs;
use rustc::ty;

use super::EvalContext;
use error::EvalResult;
use memory::Pointer;
use super::terminator::{get_impl_method, ImplMethod};

impl<'a, 'tcx> EvalContext<'a, 'tcx> {
    /// Creates a dynamic vtable for the given type and vtable origin. This is used only for
    /// objects.
    ///
    /// The `trait_ref` encodes the erased self type. Hence if we are
    /// making an object `Foo<Trait>` from a value of type `Foo<T>`, then
    /// `trait_ref` would map `T:Trait`.
    pub fn get_vtable(&mut self, trait_ref: ty::PolyTraitRef<'tcx>) -> EvalResult<'tcx, Pointer> {
        let tcx = self.tcx;

        debug!("get_vtable(trait_ref={:?})", trait_ref);

        let methods: Vec<_> = traits::supertraits(tcx, trait_ref).flat_map(|trait_ref| {
            match self.fulfill_obligation(trait_ref) {
                // Should default trait error here?
                traits::VtableDefaultImpl(_) |
                traits::VtableBuiltin(_) => {
                    Vec::new().into_iter()
                }
                traits::VtableImpl(
                    traits::VtableImplData {
                        impl_def_id: id,
                        substs,
                        nested: _ }) => {
                    self.get_vtable_methods(id, substs)
                        .into_iter()
                        .map(|opt_mth| opt_mth.map(|mth| {
                            let fn_ty = self.tcx.item_type(mth.method.def_id);
                            let fn_ty = match fn_ty.sty {
                                ty::TyFnDef(_, _, fn_ty) => fn_ty,
                                _ => bug!("bad function type: {}", fn_ty),
                            };
                            let fn_ty = self.tcx.erase_regions(&fn_ty);
                            self.memory.create_fn_ptr(self.tcx, mth.method.def_id, mth.substs, fn_ty)
                        }))
                        .collect::<Vec<_>>()
                        .into_iter()
                }
                traits::VtableClosure(
                    traits::VtableClosureData {
                        closure_def_id,
                        substs,
                        nested: _ }) => {
                    let closure_type = self.tcx.closure_type(closure_def_id, substs);
                    vec![Some(self.memory.create_closure_ptr(self.tcx, closure_def_id, substs, closure_type))].into_iter()
                }
                traits::VtableFnPointer(
                    traits::VtableFnPointerData {
                        fn_ty,
                        nested: _ }) => {
                    match fn_ty.sty {
                        ty::TyFnDef(did, substs, bare_fn_ty) => {
                            vec![Some(self.memory.create_fn_ptr(self.tcx, did, substs, bare_fn_ty))].into_iter()
                        },
                        _ => bug!("bad VtableFnPointer fn_ty: {:?}", fn_ty),
                    }
                }
                traits::VtableObject(ref data) => {
                    // this would imply that the Self type being erased is
                    // an object type; this cannot happen because we
                    // cannot cast an unsized type into a trait object
                    bug!("cannot get vtable for an object type: {:?}",
                         data);
                }
                vtable @ traits::VtableParam(..) => {
                    bug!("resolved vtable for {:?} to bad vtable {:?} in trans",
                         trait_ref,
                         vtable);
                }
            }
        }).collect();

        let size = self.type_size(trait_ref.self_ty())?.expect("can't create a vtable for an unsized type");
        let align = self.type_align(trait_ref.self_ty())?;

        let ptr_size = self.memory.pointer_size();
        let vtable = self.memory.allocate(ptr_size * (3 + methods.len() as u64), ptr_size)?;

        // in case there is no drop function to be called, this still needs to be initialized
        self.memory.write_usize(vtable, 0)?;
        if let ty::TyAdt(adt_def, substs) = trait_ref.self_ty().sty {
            if let Some(drop_def_id) = adt_def.destructor() {
                let fn_ty = match  self.tcx.item_type(drop_def_id).sty {
                    ty::TyFnDef(_, _, fn_ty) => self.tcx.erase_regions(&fn_ty),
                    _ => bug!("drop method is not a TyFnDef"),
                };
                let fn_ptr = self.memory.create_fn_ptr(self.tcx, drop_def_id, substs, fn_ty);
                self.memory.write_ptr(vtable, fn_ptr)?;
            }
        }

        self.memory.write_usize(vtable.offset(ptr_size), size)?;
        self.memory.write_usize(vtable.offset((ptr_size * 2)), align)?;

        for (i, method) in methods.into_iter().enumerate() {
            if let Some(method) = method {
                self.memory.write_ptr(vtable.offset(ptr_size * (3 + i as u64)), method)?;
            }
        }

        self.memory.freeze(vtable.alloc_id)?;

        Ok(vtable)
    }

    fn get_vtable_methods(&mut self, impl_id: DefId, substs: &'tcx Substs<'tcx>) -> Vec<Option<ImplMethod<'tcx>>> {
        debug!("get_vtable_methods(impl_id={:?}, substs={:?}", impl_id, substs);

        let trait_id = match self.tcx.impl_trait_ref(impl_id) {
            Some(t_id) => t_id.def_id,
            None       => bug!("make_impl_vtable: don't know how to \
                                make a vtable for a type impl!")
        };

        self.tcx.populate_implementations_for_trait_if_necessary(trait_id);

        self.tcx
            .associated_items(trait_id)
            // Filter out non-method items.
            .filter_map(|trait_method_type| {
                if trait_method_type.kind != ty::AssociatedKind::Method {
                    return None;
                }
                debug!("get_vtable_methods: trait_method_type={:?}",
                       trait_method_type);

                let name = trait_method_type.name;

                // Some methods cannot be called on an object; skip those.
                if !self.tcx.is_vtable_safe_method(trait_id, &trait_method_type) {
                    debug!("get_vtable_methods: not vtable safe");
                    return Some(None);
                }

                debug!("get_vtable_methods: trait_method_type={:?}",
                       trait_method_type);

                // the method may have some early-bound lifetimes, add
                // regions for those
                let method_substs = Substs::for_item(self.tcx, trait_method_type.def_id,
                                                     |_, _| self.tcx.mk_region(ty::ReErased),
                                                     |_, _| self.tcx.types.err);

                // The substitutions we have are on the impl, so we grab
                // the method type from the impl to substitute into.
                let mth = get_impl_method(self.tcx, method_substs, impl_id, substs, name);

                debug!("get_vtable_methods: mth={:?}", mth);

                // If this is a default method, it's possible that it
                // relies on where clauses that do not hold for this
                // particular set of type parameters. Note that this
                // method could then never be called, so we do not want to
                // try and trans it, in that case. Issue #23435.
                if mth.is_provided {
                    let predicates = self.tcx.item_predicates(trait_method_type.def_id).instantiate_own(self.tcx, mth.substs);
                    if !self.normalize_and_test_predicates(predicates.predicates) {
                        debug!("get_vtable_methods: predicates do not hold");
                        return Some(None);
                    }
                }

                Some(Some(mth))
            })
            .collect()
    }

    /// Normalizes the predicates and checks whether they hold.  If this
    /// returns false, then either normalize encountered an error or one
    /// of the predicates did not hold. Used when creating vtables to
    /// check for unsatisfiable methods.
    fn normalize_and_test_predicates(&mut self, predicates: Vec<ty::Predicate<'tcx>>) -> bool {
        debug!("normalize_and_test_predicates(predicates={:?})",
               predicates);

        self.tcx.infer_ctxt(None, None, Reveal::All).enter(|infcx| {
            let mut selcx = SelectionContext::new(&infcx);
            let mut fulfill_cx = traits::FulfillmentContext::new();
            let cause = traits::ObligationCause::dummy();
            let traits::Normalized { value: predicates, obligations } =
                traits::normalize(&mut selcx, cause.clone(), &predicates);
            for obligation in obligations {
                fulfill_cx.register_predicate_obligation(&infcx, obligation);
            }
            for predicate in predicates {
                let obligation = traits::Obligation::new(cause.clone(), predicate);
                fulfill_cx.register_predicate_obligation(&infcx, obligation);
            }

            fulfill_cx.select_all_or_error(&infcx).is_ok()
        })
    }
}
<commit_msg>Clean up vtable imports.<commit_after>use rustc::hir::def_id::DefId;
use rustc::traits::{self, Reveal, SelectionContext};
use rustc::ty::subst::Substs;
use rustc::ty;

use error::EvalResult;
use eval_context::EvalContext;
use memory::Pointer;
use terminator::{get_impl_method, ImplMethod};

impl<'a, 'tcx> EvalContext<'a, 'tcx> {
    /// Creates a dynamic vtable for the given type and vtable origin. This is used only for
    /// objects.
    ///
    /// The `trait_ref` encodes the erased self type. Hence if we are
    /// making an object `Foo<Trait>` from a value of type `Foo<T>`, then
    /// `trait_ref` would map `T:Trait`.
    pub fn get_vtable(&mut self, trait_ref: ty::PolyTraitRef<'tcx>) -> EvalResult<'tcx, Pointer> {
        let tcx = self.tcx;

        debug!("get_vtable(trait_ref={:?})", trait_ref);

        let methods: Vec<_> = traits::supertraits(tcx, trait_ref).flat_map(|trait_ref| {
            match self.fulfill_obligation(trait_ref) {
                // Should default trait error here?
                traits::VtableDefaultImpl(_) |
                traits::VtableBuiltin(_) => {
                    Vec::new().into_iter()
                }
                traits::VtableImpl(
                    traits::VtableImplData {
                        impl_def_id: id,
                        substs,
                        nested: _ }) => {
                    self.get_vtable_methods(id, substs)
                        .into_iter()
                        .map(|opt_mth| opt_mth.map(|mth| {
                            let fn_ty = self.tcx.item_type(mth.method.def_id);
                            let fn_ty = match fn_ty.sty {
                                ty::TyFnDef(_, _, fn_ty) => fn_ty,
                                _ => bug!("bad function type: {}", fn_ty),
                            };
                            let fn_ty = self.tcx.erase_regions(&fn_ty);
                            self.memory.create_fn_ptr(self.tcx, mth.method.def_id, mth.substs, fn_ty)
                        }))
                        .collect::<Vec<_>>()
                        .into_iter()
                }
                traits::VtableClosure(
                    traits::VtableClosureData {
                        closure_def_id,
                        substs,
                        nested: _ }) => {
                    let closure_type = self.tcx.closure_type(closure_def_id, substs);
                    vec![Some(self.memory.create_closure_ptr(self.tcx, closure_def_id, substs, closure_type))].into_iter()
                }
                traits::VtableFnPointer(
                    traits::VtableFnPointerData {
                        fn_ty,
                        nested: _ }) => {
                    match fn_ty.sty {
                        ty::TyFnDef(did, substs, bare_fn_ty) => {
                            vec![Some(self.memory.create_fn_ptr(self.tcx, did, substs, bare_fn_ty))].into_iter()
                        },
                        _ => bug!("bad VtableFnPointer fn_ty: {:?}", fn_ty),
                    }
                }
                traits::VtableObject(ref data) => {
                    // this would imply that the Self type being erased is
                    // an object type; this cannot happen because we
                    // cannot cast an unsized type into a trait object
                    bug!("cannot get vtable for an object type: {:?}",
                         data);
                }
                vtable @ traits::VtableParam(..) => {
                    bug!("resolved vtable for {:?} to bad vtable {:?} in trans",
                         trait_ref,
                         vtable);
                }
            }
        }).collect();

        let size = self.type_size(trait_ref.self_ty())?.expect("can't create a vtable for an unsized type");
        let align = self.type_align(trait_ref.self_ty())?;

        let ptr_size = self.memory.pointer_size();
        let vtable = self.memory.allocate(ptr_size * (3 + methods.len() as u64), ptr_size)?;

        // in case there is no drop function to be called, this still needs to be initialized
        self.memory.write_usize(vtable, 0)?;
        if let ty::TyAdt(adt_def, substs) = trait_ref.self_ty().sty {
            if let Some(drop_def_id) = adt_def.destructor() {
                let fn_ty = match  self.tcx.item_type(drop_def_id).sty {
                    ty::TyFnDef(_, _, fn_ty) => self.tcx.erase_regions(&fn_ty),
                    _ => bug!("drop method is not a TyFnDef"),
                };
                let fn_ptr = self.memory.create_fn_ptr(self.tcx, drop_def_id, substs, fn_ty);
                self.memory.write_ptr(vtable, fn_ptr)?;
            }
        }

        self.memory.write_usize(vtable.offset(ptr_size), size)?;
        self.memory.write_usize(vtable.offset((ptr_size * 2)), align)?;

        for (i, method) in methods.into_iter().enumerate() {
            if let Some(method) = method {
                self.memory.write_ptr(vtable.offset(ptr_size * (3 + i as u64)), method)?;
            }
        }

        self.memory.freeze(vtable.alloc_id)?;

        Ok(vtable)
    }

    fn get_vtable_methods(&mut self, impl_id: DefId, substs: &'tcx Substs<'tcx>) -> Vec<Option<ImplMethod<'tcx>>> {
        debug!("get_vtable_methods(impl_id={:?}, substs={:?}", impl_id, substs);

        let trait_id = match self.tcx.impl_trait_ref(impl_id) {
            Some(t_id) => t_id.def_id,
            None       => bug!("make_impl_vtable: don't know how to \
                                make a vtable for a type impl!")
        };

        self.tcx.populate_implementations_for_trait_if_necessary(trait_id);

        self.tcx
            .associated_items(trait_id)
            // Filter out non-method items.
            .filter_map(|trait_method_type| {
                if trait_method_type.kind != ty::AssociatedKind::Method {
                    return None;
                }
                debug!("get_vtable_methods: trait_method_type={:?}",
                       trait_method_type);

                let name = trait_method_type.name;

                // Some methods cannot be called on an object; skip those.
                if !self.tcx.is_vtable_safe_method(trait_id, &trait_method_type) {
                    debug!("get_vtable_methods: not vtable safe");
                    return Some(None);
                }

                debug!("get_vtable_methods: trait_method_type={:?}",
                       trait_method_type);

                // the method may have some early-bound lifetimes, add
                // regions for those
                let method_substs = Substs::for_item(self.tcx, trait_method_type.def_id,
                                                     |_, _| self.tcx.mk_region(ty::ReErased),
                                                     |_, _| self.tcx.types.err);

                // The substitutions we have are on the impl, so we grab
                // the method type from the impl to substitute into.
                let mth = get_impl_method(self.tcx, method_substs, impl_id, substs, name);

                debug!("get_vtable_methods: mth={:?}", mth);

                // If this is a default method, it's possible that it
                // relies on where clauses that do not hold for this
                // particular set of type parameters. Note that this
                // method could then never be called, so we do not want to
                // try and trans it, in that case. Issue #23435.
                if mth.is_provided {
                    let predicates = self.tcx.item_predicates(trait_method_type.def_id).instantiate_own(self.tcx, mth.substs);
                    if !self.normalize_and_test_predicates(predicates.predicates) {
                        debug!("get_vtable_methods: predicates do not hold");
                        return Some(None);
                    }
                }

                Some(Some(mth))
            })
            .collect()
    }

    /// Normalizes the predicates and checks whether they hold.  If this
    /// returns false, then either normalize encountered an error or one
    /// of the predicates did not hold. Used when creating vtables to
    /// check for unsatisfiable methods.
    fn normalize_and_test_predicates(&mut self, predicates: Vec<ty::Predicate<'tcx>>) -> bool {
        debug!("normalize_and_test_predicates(predicates={:?})",
               predicates);

        self.tcx.infer_ctxt(None, None, Reveal::All).enter(|infcx| {
            let mut selcx = SelectionContext::new(&infcx);
            let mut fulfill_cx = traits::FulfillmentContext::new();
            let cause = traits::ObligationCause::dummy();
            let traits::Normalized { value: predicates, obligations } =
                traits::normalize(&mut selcx, cause.clone(), &predicates);
            for obligation in obligations {
                fulfill_cx.register_predicate_obligation(&infcx, obligation);
            }
            for predicate in predicates {
                let obligation = traits::Obligation::new(cause.clone(), predicate);
                fulfill_cx.register_predicate_obligation(&infcx, obligation);
            }

            fulfill_cx.select_all_or_error(&infcx).is_ok()
        })
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.
extern crate zmq;
extern crate rand;
extern crate tempfile;

use zmq::{Error, Message, Context, SNDMORE};
use std::ops::Deref;
use std::thread;
use std::env;
use std::fs::File;
use std::io::{Read,Write, Seek, SeekFrom};
use std::path::Path;
// use std::str;
use std::process::Command;
use rand::{random};

pub trait Worker {
  // fn work(&self, &Message) -> Option<Message>;
  fn convert(&self, &Path) -> Option<File>;
  fn message_size(&self) -> usize;
  fn service(&self) -> String;
  fn source(&self) -> String;
  fn sink(&self) -> String;

  fn start(&self, limit : Option<i32>) -> Result<(), Error> {
    let mut work_counter = 0;
    // Connect to a task ventilator
    let mut context_source = Context::new();
    let mut source = context_source.socket(zmq::DEALER).unwrap();
    let identity : String = (0..10).map(|_| rand::random::<u8>() as char).collect();
    source.set_identity(identity.as_bytes()).unwrap();

    assert!(source.connect(&self.source()).is_ok());
    // Connect to a task sink
    let mut context_sink = Context::new();
    let mut sink = context_sink.socket(zmq::PUSH).unwrap();
    assert!(sink.connect(&self.sink()).is_ok());
    // Work in perpetuity
    loop {
      let mut taskid_msg = Message::new().unwrap();
      let mut recv_msg = Message::new().unwrap();

      source.send_str(&self.service(), 0).unwrap();
      source.recv(&mut taskid_msg, 0).unwrap();
      let taskid = taskid_msg.as_str().unwrap();

      // Prepare a File for the input
      let input_filepath = env::temp_dir().to_str().unwrap().to_string() + "/" + taskid ;
      let mut file = File::create(input_filepath.clone()).unwrap();
      loop {
        source.recv(&mut recv_msg, 0).unwrap();

        file.write(recv_msg.deref()).unwrap();
        if !source.get_rcvmore().unwrap() {
          break;
        }
      }

      file.seek(SeekFrom::Start(0)).unwrap();
      let file_opt = self.convert(Path::new(&input_filepath));
      if file_opt.is_some() {
        let mut converted_file = file_opt.unwrap();
        sink.send_str(&self.service(), SNDMORE).unwrap();
        sink.send_str(taskid, SNDMORE).unwrap();
        loop {
          // Stream converted data via zmq
          let message_size = self.message_size();
          let mut data = vec![0; message_size];
          let size = converted_file.read(&mut data).unwrap();
          data.truncate(size);
          if size < message_size {
            // If exhausted, send the last frame
            sink.send(&data,0).unwrap();
            // And terminate
            break;
          } else {
            // If more to go, send the frame and indicate there's more to come
            sink.send(&data,SNDMORE).unwrap();
          }
        }
      }
      else {
        // If there was nothing to do, retry a minute later
        thread::sleep_ms(60000);
        continue
      }

      work_counter += 1;
      match limit {
        Some(upper_bound) => {
          if work_counter >= upper_bound {
            // Give enough time to complete the Final job.
            thread::sleep_ms(500);
            break;
          }
        },
        None => {}
      };
    }
    Ok(())
  }
}
pub struct EchoWorker {
  pub service : String,
  pub version : f32,
  pub message_size : usize,
  pub source : String,
  pub sink : String
}
impl Default for EchoWorker {
  fn default() -> EchoWorker {
    EchoWorker {
      service: "echo_service".to_string(),
      version: 0.1,
      message_size: 100000,
      source: "tcp://localhost:5555".to_string(),
      sink: "tcp://localhost:5556".to_string()
    }
  }
}
impl Worker for EchoWorker {
  fn service(&self) -> String {self.service.clone()}
  fn source(&self) -> String {self.source.clone()}
  fn sink(&self) -> String {self.sink.clone()}
  fn message_size(&self) -> usize {self.message_size.clone()}

  fn convert(&self, path : &Path) -> Option<File> {
    Some(File::open(path).unwrap())
  }
}

pub struct TexToHtmlWorker {
  pub service : String,
  pub version : f32,
  pub message_size : usize,
  pub source : String,
  pub sink : String
}
impl Default for TexToHtmlWorker {
  fn default() -> TexToHtmlWorker {
    TexToHtmlWorker {
      service: "tex_to_html".to_string(),
      version: 0.1,
      message_size: 100000,
      source: "tcp://localhost:5555".to_string(),
      sink: "tcp://localhost:5556".to_string()
    }
  }
}
impl Worker for TexToHtmlWorker {
  fn service(&self) -> String {self.service.clone()}
  fn source(&self) -> String {self.source.clone()}
  fn sink(&self) -> String {self.sink.clone()}
  fn message_size(&self) -> usize {self.message_size.clone()}

  fn convert(&self, path : &Path) -> Option<File> {
    let name = path.file_stem().unwrap().to_str().unwrap();
    let destination_path = env::temp_dir().to_str().unwrap().to_string() + "/" +name+ ".zip";

    Command::new("latexmlc")
      .arg("--whatsin")
      .arg("archive")
      .arg("--whatsout")
      .arg("archive")
      .arg("--format")
      .arg("html5")
      .arg("--pmml")
      .arg("--cmml")
      .arg("--preload")
      .arg("[ids]latexml.sty")
      .arg("--css")
      .arg("http://latexml.mathweb.org/css/external/LaTeXML.css")
      .arg("--nodefaultresources")
      .arg("--inputencoding")
      .arg("iso-8859-1")
      .arg("--timeout")
      .arg("300")
      .arg("--log")
      .arg(name.to_string() + ".log")
      .arg("--destination")
      .arg(destination_path.clone())
      .arg(path.clone())
      .output()
      .unwrap_or_else(|e| { panic!("failed to execute process: {}", e) });

    // println!("Dest: {:?}", destination_path);
    // println!("Log: {:?}", str::from_utf8(&output.stderr));
    Some(File::open(destination_path.clone()).unwrap())
  }
}
<commit_msg>simplifying filename conventions<commit_after>// Copyright 2015 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.
extern crate zmq;
extern crate rand;
extern crate tempfile;

use zmq::{Error, Message, Context, SNDMORE};
use std::ops::Deref;
use std::thread;
use std::env;
use std::fs::File;
use std::io::{Read,Write, Seek, SeekFrom};
use std::path::Path;
// use std::str;
use std::process::Command;
use rand::{random};

pub trait Worker {
  // fn work(&self, &Message) -> Option<Message>;
  fn convert(&self, &Path) -> Option<File>;
  fn message_size(&self) -> usize;
  fn service(&self) -> String;
  fn source(&self) -> String;
  fn sink(&self) -> String;

  fn start(&self, limit : Option<i32>) -> Result<(), Error> {
    let mut work_counter = 0;
    // Connect to a task ventilator
    let mut context_source = Context::new();
    let mut source = context_source.socket(zmq::DEALER).unwrap();
    let identity : String = (0..10).map(|_| rand::random::<u8>() as char).collect();
    source.set_identity(identity.as_bytes()).unwrap();

    assert!(source.connect(&self.source()).is_ok());
    // Connect to a task sink
    let mut context_sink = Context::new();
    let mut sink = context_sink.socket(zmq::PUSH).unwrap();
    assert!(sink.connect(&self.sink()).is_ok());
    // Work in perpetuity
    loop {
      let mut taskid_msg = Message::new().unwrap();
      let mut recv_msg = Message::new().unwrap();

      source.send_str(&self.service(), 0).unwrap();
      source.recv(&mut taskid_msg, 0).unwrap();
      let taskid = taskid_msg.as_str().unwrap();

      // Prepare a File for the input
      let input_filepath = env::temp_dir().to_str().unwrap().to_string() + "/" + taskid ;
      let mut file = File::create(input_filepath.clone()).unwrap();
      loop {
        source.recv(&mut recv_msg, 0).unwrap();

        file.write(recv_msg.deref()).unwrap();
        if !source.get_rcvmore().unwrap() {
          break;
        }
      }

      file.seek(SeekFrom::Start(0)).unwrap();
      let file_opt = self.convert(Path::new(&input_filepath));
      if file_opt.is_some() {
        let mut converted_file = file_opt.unwrap();
        sink.send_str(&self.service(), SNDMORE).unwrap();
        sink.send_str(taskid, SNDMORE).unwrap();
        loop {
          // Stream converted data via zmq
          let message_size = self.message_size();
          let mut data = vec![0; message_size];
          let size = converted_file.read(&mut data).unwrap();
          data.truncate(size);
          if size < message_size {
            // If exhausted, send the last frame
            sink.send(&data,0).unwrap();
            // And terminate
            break;
          } else {
            // If more to go, send the frame and indicate there's more to come
            sink.send(&data,SNDMORE).unwrap();
          }
        }
      }
      else {
        // If there was nothing to do, retry a minute later
        thread::sleep_ms(60000);
        continue
      }

      work_counter += 1;
      match limit {
        Some(upper_bound) => {
          if work_counter >= upper_bound {
            // Give enough time to complete the Final job.
            thread::sleep_ms(500);
            break;
          }
        },
        None => {}
      };
    }
    Ok(())
  }
}
pub struct EchoWorker {
  pub service : String,
  pub version : f32,
  pub message_size : usize,
  pub source : String,
  pub sink : String
}
impl Default for EchoWorker {
  fn default() -> EchoWorker {
    EchoWorker {
      service: "echo_service".to_string(),
      version: 0.1,
      message_size: 100000,
      source: "tcp://localhost:5555".to_string(),
      sink: "tcp://localhost:5556".to_string()
    }
  }
}
impl Worker for EchoWorker {
  fn service(&self) -> String {self.service.clone()}
  fn source(&self) -> String {self.source.clone()}
  fn sink(&self) -> String {self.sink.clone()}
  fn message_size(&self) -> usize {self.message_size.clone()}

  fn convert(&self, path : &Path) -> Option<File> {
    Some(File::open(path).unwrap())
  }
}

pub struct TexToHtmlWorker {
  pub service : String,
  pub version : f32,
  pub message_size : usize,
  pub source : String,
  pub sink : String
}
impl Default for TexToHtmlWorker {
  fn default() -> TexToHtmlWorker {
    TexToHtmlWorker {
      service: "tex_to_html".to_string(),
      version: 0.1,
      message_size: 100000,
      source: "tcp://localhost:5555".to_string(),
      sink: "tcp://localhost:5556".to_string()
    }
  }
}
impl Worker for TexToHtmlWorker {
  fn service(&self) -> String {self.service.clone()}
  fn source(&self) -> String {self.source.clone()}
  fn sink(&self) -> String {self.sink.clone()}
  fn message_size(&self) -> usize {self.message_size.clone()}

  fn convert(&self, path : &Path) -> Option<File> {
    let name = path.file_stem().unwrap().to_str().unwrap();
    let destination_path = env::temp_dir().to_str().unwrap().to_string() + "/" +name+ ".zip";

    Command::new("latexmlc")
      .arg("--whatsin")
      .arg("archive")
      .arg("--whatsout")
      .arg("archive")
      .arg("--format")
      .arg("html5")
      .arg("--pmml")
      .arg("--cmml")
      .arg("--preload")
      .arg("[ids]latexml.sty")
      .arg("--css")
      .arg("http://latexml.mathweb.org/css/external/LaTeXML.css")
      .arg("--nodefaultresources")
      .arg("--inputencoding")
      .arg("iso-8859-1")
      .arg("--timeout")
      .arg("300")
      .arg("--log")
      .arg("cortex.log")
      .arg("--destination")
      .arg(destination_path.clone())
      .arg(path.clone())
      .output()
      .unwrap_or_else(|e| { panic!("failed to execute process: {}", e) });

    // println!("Dest: {:?}", destination_path);
    // println!("Log: {:?}", str::from_utf8(&output.stderr));
    Some(File::open(destination_path.clone()).unwrap())
  }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

// Shader key is in "name feature,feature" format.
// File name needs to be formatted as "name_feature_feature".
fn shader_file(shader_key: &str) -> String {
    shader_key.replace(' ', "_").replace(',', "_")
}

fn write_load_shader(shader_keys: &[String]) {
    let mut load_shader = String::new();
    for s in shader_keys {
        let _ = write!(load_shader, "#include \"{}.h\"\n", shader_file(s));
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in shader_keys {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{}\")) {{ return {}_program::loader; }}\n",
                       s, shader_file(s));
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::from("#define SWGL 1\n#define __VERSION__ 150\n");
    let _ = write!(imported, "#define WR_MAX_VERTEX_TEXTURE_WIDTH {}U\n",
                   webrender_build::MAX_VERTEX_TEXTURE_WIDTH);

    let (basename, features) =
        shader_key.split_at(shader_key.find(' ').unwrap_or(shader_key.len()));
    if !features.is_empty() {
        for feature in features.trim().split(',') {
            let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
        }
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_file(shader_key);

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    build.no_default_flags(true);
    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/EP");
            if tool.path().to_str().map_or(false, |p| p.contains("clang")) {
                build.flag("/clang:-undef");
            } else {
                build.flag("/u");
            }
        } else {
            build.flag("-xc").flag("-P").flag("-undef");
        }
    }
    // Use SWGLPP target to avoid pulling CFLAGS/CXXFLAGS.
    build.target("SWGLPP");
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = match std::env::var("MOZ_SRC") {
        Ok(dir) => dir + "/gfx/wr/webrender/res",
        Err(_) => std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res",
    };

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING |
        ShaderFeatureFlags::ADVANCED_BLEND_EQUATION |
        ShaderFeatureFlags::DEBUG;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{} {}", name, f) }
        }));
    }

    shaders.sort();

    // We need to ensure that the C preprocessor does not pull compiler flags from
    // the host or target environment. Set up a SWGLPP target with empty flags to
    // work around this.
    if let Ok(target) = std::env::var("TARGET") {
        if let Ok(cc) = std::env::var(format!("CC_{}", target))
                        .or(std::env::var(format!("CC_{}", target.replace("-", "_")))) {
            std::env::set_var("CC_SWGLPP", cc);
        }
    }
    std::env::set_var("CFLAGS_SWGLPP", "");

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/blend.h");
    println!("cargo:rerun-if-changed=src/composite.h");
    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/rasterize.h");
    println!("cargo:rerun-if-changed=src/swgl_ext.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    let mut build = cc::Build::new();
    build.cpp(true);

    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/std:c++17")
                 .flag("/EHs-")
                 .flag("/GR-")
                 .flag("/UMOZILLA_CONFIG_H");
        } else {
            build.flag("-std=c++17")
                 .flag("-fno-exceptions")
                 .flag("-fno-rtti")
                 .flag("-fno-math-errno")
                 .flag("-UMOZILLA_CONFIG_H");
        }
        // SWGL relies heavily on inlining for performance so override -Oz with -O2
        if tool.args().contains(&"-Oz".into()) {
            build.flag("-O2");
        }

        // Most GLSL compilers assume something like fast-math so we turn it on.
        // However, reciprocal division makes it so 1/1 = 0.999994 which can produce a lot of fuzz
        // in reftests and the use of reciprocal instructions usually involves a refinement step
        // which bloats our already bloated code. Further, our shader code is sufficiently parallel
        // that we're more likely to be throughput bound vs latency bound. Having fewer
        // instructions makes things easier on the processor and in places where it matters we can
        // probably explicitly use reciprocal instructions and avoid the refinement step.
        if tool.is_like_msvc() {
            build.flag("/fp:fast")
                 .flag("-Xclang")
                 .flag("-mrecip=none");
        } else if tool.is_like_clang() {
            // gcc only supports -mrecip=none on some targets so to keep
            // things simple we don't use -ffast-math with gcc at all
            build.flag("-ffast-math")
                 .flag("-mrecip=none");
        }
    }

    build.file("src/gl.cc")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<commit_msg>Bug 1751331 - Remove unnecessary use of MOZ_SRC. r=gfx-reviewers,lsalzman<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

// Shader key is in "name feature,feature" format.
// File name needs to be formatted as "name_feature_feature".
fn shader_file(shader_key: &str) -> String {
    shader_key.replace(' ', "_").replace(',', "_")
}

fn write_load_shader(shader_keys: &[String]) {
    let mut load_shader = String::new();
    for s in shader_keys {
        let _ = write!(load_shader, "#include \"{}.h\"\n", shader_file(s));
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in shader_keys {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{}\")) {{ return {}_program::loader; }}\n",
                       s, shader_file(s));
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::from("#define SWGL 1\n#define __VERSION__ 150\n");
    let _ = write!(imported, "#define WR_MAX_VERTEX_TEXTURE_WIDTH {}U\n",
                   webrender_build::MAX_VERTEX_TEXTURE_WIDTH);

    let (basename, features) =
        shader_key.split_at(shader_key.find(' ').unwrap_or(shader_key.len()));
    if !features.is_empty() {
        for feature in features.trim().split(',') {
            let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
        }
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_file(shader_key);

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    build.no_default_flags(true);
    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/EP");
            if tool.path().to_str().map_or(false, |p| p.contains("clang")) {
                build.flag("/clang:-undef");
            } else {
                build.flag("/u");
            }
        } else {
            build.flag("-xc").flag("-P").flag("-undef");
        }
    }
    // Use SWGLPP target to avoid pulling CFLAGS/CXXFLAGS.
    build.target("SWGLPP");
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res";

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING |
        ShaderFeatureFlags::ADVANCED_BLEND_EQUATION |
        ShaderFeatureFlags::DEBUG;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{} {}", name, f) }
        }));
    }

    shaders.sort();

    // We need to ensure that the C preprocessor does not pull compiler flags from
    // the host or target environment. Set up a SWGLPP target with empty flags to
    // work around this.
    if let Ok(target) = std::env::var("TARGET") {
        if let Ok(cc) = std::env::var(format!("CC_{}", target))
                        .or(std::env::var(format!("CC_{}", target.replace("-", "_")))) {
            std::env::set_var("CC_SWGLPP", cc);
        }
    }
    std::env::set_var("CFLAGS_SWGLPP", "");

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/blend.h");
    println!("cargo:rerun-if-changed=src/composite.h");
    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/rasterize.h");
    println!("cargo:rerun-if-changed=src/swgl_ext.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    let mut build = cc::Build::new();
    build.cpp(true);

    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/std:c++17")
                 .flag("/EHs-")
                 .flag("/GR-")
                 .flag("/UMOZILLA_CONFIG_H");
        } else {
            build.flag("-std=c++17")
                 .flag("-fno-exceptions")
                 .flag("-fno-rtti")
                 .flag("-fno-math-errno")
                 .flag("-UMOZILLA_CONFIG_H");
        }
        // SWGL relies heavily on inlining for performance so override -Oz with -O2
        if tool.args().contains(&"-Oz".into()) {
            build.flag("-O2");
        }

        // Most GLSL compilers assume something like fast-math so we turn it on.
        // However, reciprocal division makes it so 1/1 = 0.999994 which can produce a lot of fuzz
        // in reftests and the use of reciprocal instructions usually involves a refinement step
        // which bloats our already bloated code. Further, our shader code is sufficiently parallel
        // that we're more likely to be throughput bound vs latency bound. Having fewer
        // instructions makes things easier on the processor and in places where it matters we can
        // probably explicitly use reciprocal instructions and avoid the refinement step.
        if tool.is_like_msvc() {
            build.flag("/fp:fast")
                 .flag("-Xclang")
                 .flag("-mrecip=none");
        } else if tool.is_like_clang() {
            // gcc only supports -mrecip=none on some targets so to keep
            // things simple we don't use -ffast-math with gcc at all
            build.flag("-ffast-math")
                 .flag("-mrecip=none");
        }
    }

    build.file("src/gl.cc")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<|endoftext|>


<commit_before>//! `log` crate adapter for slog-rs
//!
//! This crate provides two way compatibility with Rust standard `log` crate.
//!
//! ### `log` -> `slog`
//!
//! After calling `init()` `slog-stdlog` will take a role of `log` crate
//! back-end, forwarding all the `log` logging to `slog_scope::logger()`.
//! In other words, any `log` crate logging statement will behave like it was `slog`
//! logging statement executed with logger returned by `slog_scope::logger()`.
//!
//! See documentation of `slog-scope` for more information about logging scopes.
//!
//! See [`init` documentation](fn.init.html) for an example.
//!
//! ### `slog` -> `log`
//!
//! `StdLog` is `slog::Drain` that will pass all `Record`s passing through it to
//! `log` crate just like they were crated with `log` crate logging macros in
//! the first place.
//!
//! ## `slog-scope`
//!
//! Since `log` does not have any form of context, and does not support `Logger`
//! `slog-stdlog` relies on "logging scopes" to establish it.
//!
//! You must set up logging context for `log` -> `slog` via `slog_scope::scope`
//! or `slog_scope::set_global_logger`. Setting a global logger upfront via
//! `slog_scope::set_global_logger` is highly recommended.
//!
//! Note: Since `slog-stdlog` v2, unlike previous releases, `slog-stdlog` uses
//! logging scopes provided by `slog-scope` crate instead of it's own.
//!
//! Refer to `slog-scope` crate documentation for more information.
//!
//! ### Warning
//!
//! Be careful when using both methods at the same time, as a loop can be easily
//! created: `log` -> `slog` -> `log` -> ...
//!
//! ## Compile-time log level filtering
//!
//! For filtering `debug!` and other `log` statements at compile-time, configure
//! the features on the `log` crate in your `Cargo.toml`:
//!
//! ```norust
//! log = { version = "*", features = ["max_level_trace", "release_max_level_warn"] }
//! ```
#![warn(missing_docs)]

extern crate log;

#[cfg(feature = "kv_unstable")]
mod kv;

use std::{fmt, io};
use slog::{Level, KV, b};

struct Logger;

fn log_to_slog_level(level: log::Level) -> Level {
    match level {
        log::Level::Trace => Level::Trace,
        log::Level::Debug => Level::Debug,
        log::Level::Info => Level::Info,
        log::Level::Warn => Level::Warning,
        log::Level::Error => Level::Error,
    }
}

fn record_as_location(r: &log::Record) -> slog::RecordLocation {
    let module = r.module_path_static().unwrap_or("<unknown>");
    let file = r.file_static().unwrap_or("<unknown>");
    let line = r.line().unwrap_or_default();

    slog::RecordLocation {
        file,
        line,
        column: 0,
        function: "",
        module,
    }
}

impl log::Log for Logger {
    fn enabled(&self, _: &log::Metadata) -> bool {
        true
    }

    fn log(&self, r: &log::Record) {
        let level = log_to_slog_level(r.metadata().level());

        let args = r.args();
        let target = r.target();
        let location = &record_as_location(r);
        let s = slog::RecordStatic {
            location,
            level,
            tag: target,
        };
        #[cfg(feature = "kv_unstable")]
        {
             let key_values = r.key_values();
             let mut visitor = kv::Visitor::new();
             key_values.visit(&mut visitor).unwrap();
             slog_scope::with_logger(|logger| logger.log(&slog::Record::new(&s, args, b!(visitor))))
        }
        #[cfg(not(feature = "kv_unstable"))]
        slog_scope::with_logger(|logger| logger.log(&slog::Record::new(&s, args, b!())))
    }

    fn flush(&self) {}
}

/// Register `slog-stdlog` as `log` backend.
///
/// This will pass all logging statements crated with `log`
/// crate to current `slog-scope::logger()`.
///
/// ```
/// #[macro_use]
/// extern crate log;
/// #[macro_use(slog_o, slog_kv)]
/// extern crate slog;
/// extern crate slog_stdlog;
/// extern crate slog_scope;
/// extern crate slog_term;
/// extern crate slog_async;
///
/// use slog::Drain;
///
/// fn main() {
///     let decorator = slog_term::TermDecorator::new().build();
///     let drain = slog_term::FullFormat::new(decorator).build().fuse();
///     let drain = slog_async::Async::new(drain).build().fuse();
///     let logger = slog::Logger::root(drain, slog_o!("version" => env!("CARGO_PKG_VERSION")));
///
///     let _scope_guard = slog_scope::set_global_logger(logger);
///     let _log_guard = slog_stdlog::init().unwrap();
///     // Note: this `info!(...)` macro comes from `log` crate
///     info!("standard logging redirected to slog");
/// }
/// ```
pub fn init() -> Result<(), log::SetLoggerError> {
    init_with_level(log::Level::max())
}

/// Register `slog-stdlog` as `log` backend.
/// Pass a log::Level to do the log filter explicitly.
///
/// This will pass all logging statements crated with `log`
/// crate to current `slog-scope::logger()`.
///
/// ```
/// #[macro_use]
/// extern crate log;
/// #[macro_use(slog_o, slog_kv)]
/// extern crate slog;
/// extern crate slog_stdlog;
/// extern crate slog_scope;
/// extern crate slog_term;
/// extern crate slog_async;
///
/// use slog::Drain;
///
/// fn main() {
///     let decorator = slog_term::TermDecorator::new().build();
///     let drain = slog_term::FullFormat::new(decorator).build().fuse();
///     let drain = slog_async::Async::new(drain).build().fuse();
///     let logger = slog::Logger::root(drain, slog_o!("version" => env!("CARGO_PKG_VERSION")));
///
///     let _scope_guard = slog_scope::set_global_logger(logger);
///     let _log_guard = slog_stdlog::init_with_level(log::Level::Error).unwrap();
///     // Note: this `info!(...)` macro comes from `log` crate
///     info!("standard logging redirected to slog");
///     error!("standard logging redirected to slog");
/// }
/// ```
pub fn init_with_level(level: log::Level) -> Result<(), log::SetLoggerError> {
    log::set_boxed_logger(Box::new(Logger))?;
    log::set_max_level(level.to_level_filter());

    Ok(())
}

/// Drain logging `Record`s into `log` crate
///
/// Any `Record` passing through this `Drain` will be forwarded
/// to `log` crate, just like it was created with `log` crate macros
/// in the first place. The message and key-value pairs will be formated
/// to be one string.
///
/// Caution needs to be taken to prevent circular loop where `Logger`
/// installed via `slog-stdlog::set_logger` would log things to a `StdLog`
/// drain, which would again log things to the global `Logger` and so on
/// leading to an infinite recursion.
pub struct StdLog;

struct LazyLogString<'a> {
    info: &'a slog::Record<'a>,
    logger_values: &'a slog::OwnedKVList,
}

impl<'a> LazyLogString<'a> {
    fn new(info: &'a slog::Record, logger_values: &'a slog::OwnedKVList) -> Self {
        LazyLogString {
            info,
            logger_values,
        }
    }
}

impl<'a> fmt::Display for LazyLogString<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.info.msg())?;

        let io = io::Cursor::new(Vec::new());
        let mut ser = KSV::new(io);

        self.logger_values
            .serialize(self.info, &mut ser)
            .map_err(|_| fmt::Error)?;
        self.info
            .kv()
            .serialize(self.info, &mut ser)
            .map_err(|_| fmt::Error)?;

        let values = ser.into_inner().into_inner();

        write!(f, "{}", String::from_utf8_lossy(&values))
    }
}

impl slog::Drain for StdLog {
    type Ok = ();
    type Err = io::Error;

    fn log(&self, info: &slog::Record, logger_values: &slog::OwnedKVList) -> io::Result<()> {
        let level = match info.level() {
            slog::Level::Critical | slog::Level::Error => log::Level::Error,
            slog::Level::Warning => log::Level::Warn,
            slog::Level::Info => log::Level::Info,
            slog::Level::Debug => log::Level::Debug,
            slog::Level::Trace => log::Level::Trace,
        };

        let mut target = info.tag();
        if target.is_empty() {
            target = info.module();
        }

        let lazy = LazyLogString::new(info, logger_values);
        /*
         * TODO: Support `log` crate key_values here.
         *
         * This requires the log/kv_unstable feature here.
         *
         * Not supporting this feature is backwards compatible
         * and it shouldn't break anything (because we've never had),
         * but is undesirable from a feature-completeness point of view.
         *
         * However, this is most likely not as powerful as slog's own
         * notion of key/value pairs, so I would humbly suggest using `slog`
         * directly if this feature is important to you ;)
         *
         * This avoids using the private log::__private_api_log api function,
         * which is just a thin wrapper around a `RecordBuilder`.
         */
        log::logger().log(
            &log::Record::builder()
                .args(format_args!("{}", lazy))
                .level(level)
                .target(target)
                .module_path_static(Some(info.module()))
                .file_static(Some(info.file()))
                .line(Some(info.line()))
                .build(),
        );

        Ok(())
    }
}

/// Key-Separator-Value serializer
struct KSV<W: io::Write> {
    io: W,
}

impl<W: io::Write> KSV<W> {
    fn new(io: W) -> Self {
        KSV { io: io }
    }

    fn into_inner(self) -> W {
        self.io
    }
}

impl<W: io::Write> slog::Serializer for KSV<W> {
    fn emit_arguments(&mut self, key: slog::Key, val: &fmt::Arguments) -> slog::Result {
        write!(self.io, ", {}: {}", key, val)?;
        Ok(())
    }
}
<commit_msg>Run `cargo fmt` on the code<commit_after>//! `log` crate adapter for slog-rs
//!
//! This crate provides two way compatibility with Rust standard `log` crate.
//!
//! ### `log` -> `slog`
//!
//! After calling `init()` `slog-stdlog` will take a role of `log` crate
//! back-end, forwarding all the `log` logging to `slog_scope::logger()`.
//! In other words, any `log` crate logging statement will behave like it was `slog`
//! logging statement executed with logger returned by `slog_scope::logger()`.
//!
//! See documentation of `slog-scope` for more information about logging scopes.
//!
//! See [`init` documentation](fn.init.html) for an example.
//!
//! ### `slog` -> `log`
//!
//! `StdLog` is `slog::Drain` that will pass all `Record`s passing through it to
//! `log` crate just like they were crated with `log` crate logging macros in
//! the first place.
//!
//! ## `slog-scope`
//!
//! Since `log` does not have any form of context, and does not support `Logger`
//! `slog-stdlog` relies on "logging scopes" to establish it.
//!
//! You must set up logging context for `log` -> `slog` via `slog_scope::scope`
//! or `slog_scope::set_global_logger`. Setting a global logger upfront via
//! `slog_scope::set_global_logger` is highly recommended.
//!
//! Note: Since `slog-stdlog` v2, unlike previous releases, `slog-stdlog` uses
//! logging scopes provided by `slog-scope` crate instead of it's own.
//!
//! Refer to `slog-scope` crate documentation for more information.
//!
//! ### Warning
//!
//! Be careful when using both methods at the same time, as a loop can be easily
//! created: `log` -> `slog` -> `log` -> ...
//!
//! ## Compile-time log level filtering
//!
//! For filtering `debug!` and other `log` statements at compile-time, configure
//! the features on the `log` crate in your `Cargo.toml`:
//!
//! ```norust
//! log = { version = "*", features = ["max_level_trace", "release_max_level_warn"] }
//! ```
#![warn(missing_docs)]

extern crate log;

#[cfg(feature = "kv_unstable")]
mod kv;

use slog::{b, Level, KV};
use std::{fmt, io};

struct Logger;

fn log_to_slog_level(level: log::Level) -> Level {
    match level {
        log::Level::Trace => Level::Trace,
        log::Level::Debug => Level::Debug,
        log::Level::Info => Level::Info,
        log::Level::Warn => Level::Warning,
        log::Level::Error => Level::Error,
    }
}

fn record_as_location(r: &log::Record) -> slog::RecordLocation {
    let module = r.module_path_static().unwrap_or("<unknown>");
    let file = r.file_static().unwrap_or("<unknown>");
    let line = r.line().unwrap_or_default();

    slog::RecordLocation {
        file,
        line,
        column: 0,
        function: "",
        module,
    }
}

impl log::Log for Logger {
    fn enabled(&self, _: &log::Metadata) -> bool {
        true
    }

    fn log(&self, r: &log::Record) {
        let level = log_to_slog_level(r.metadata().level());

        let args = r.args();
        let target = r.target();
        let location = &record_as_location(r);
        let s = slog::RecordStatic {
            location,
            level,
            tag: target,
        };
        #[cfg(feature = "kv_unstable")]
        {
            let key_values = r.key_values();
            let mut visitor = kv::Visitor::new();
            key_values.visit(&mut visitor).unwrap();
            slog_scope::with_logger(|logger| logger.log(&slog::Record::new(&s, args, b!(visitor))))
        }
        #[cfg(not(feature = "kv_unstable"))]
        slog_scope::with_logger(|logger| logger.log(&slog::Record::new(&s, args, b!())))
    }

    fn flush(&self) {}
}

/// Register `slog-stdlog` as `log` backend.
///
/// This will pass all logging statements crated with `log`
/// crate to current `slog-scope::logger()`.
///
/// ```
/// #[macro_use]
/// extern crate log;
/// #[macro_use(slog_o, slog_kv)]
/// extern crate slog;
/// extern crate slog_stdlog;
/// extern crate slog_scope;
/// extern crate slog_term;
/// extern crate slog_async;
///
/// use slog::Drain;
///
/// fn main() {
///     let decorator = slog_term::TermDecorator::new().build();
///     let drain = slog_term::FullFormat::new(decorator).build().fuse();
///     let drain = slog_async::Async::new(drain).build().fuse();
///     let logger = slog::Logger::root(drain, slog_o!("version" => env!("CARGO_PKG_VERSION")));
///
///     let _scope_guard = slog_scope::set_global_logger(logger);
///     let _log_guard = slog_stdlog::init().unwrap();
///     // Note: this `info!(...)` macro comes from `log` crate
///     info!("standard logging redirected to slog");
/// }
/// ```
pub fn init() -> Result<(), log::SetLoggerError> {
    init_with_level(log::Level::max())
}

/// Register `slog-stdlog` as `log` backend.
/// Pass a log::Level to do the log filter explicitly.
///
/// This will pass all logging statements crated with `log`
/// crate to current `slog-scope::logger()`.
///
/// ```
/// #[macro_use]
/// extern crate log;
/// #[macro_use(slog_o, slog_kv)]
/// extern crate slog;
/// extern crate slog_stdlog;
/// extern crate slog_scope;
/// extern crate slog_term;
/// extern crate slog_async;
///
/// use slog::Drain;
///
/// fn main() {
///     let decorator = slog_term::TermDecorator::new().build();
///     let drain = slog_term::FullFormat::new(decorator).build().fuse();
///     let drain = slog_async::Async::new(drain).build().fuse();
///     let logger = slog::Logger::root(drain, slog_o!("version" => env!("CARGO_PKG_VERSION")));
///
///     let _scope_guard = slog_scope::set_global_logger(logger);
///     let _log_guard = slog_stdlog::init_with_level(log::Level::Error).unwrap();
///     // Note: this `info!(...)` macro comes from `log` crate
///     info!("standard logging redirected to slog");
///     error!("standard logging redirected to slog");
/// }
/// ```
pub fn init_with_level(level: log::Level) -> Result<(), log::SetLoggerError> {
    log::set_boxed_logger(Box::new(Logger))?;
    log::set_max_level(level.to_level_filter());

    Ok(())
}

/// Drain logging `Record`s into `log` crate
///
/// Any `Record` passing through this `Drain` will be forwarded
/// to `log` crate, just like it was created with `log` crate macros
/// in the first place. The message and key-value pairs will be formated
/// to be one string.
///
/// Caution needs to be taken to prevent circular loop where `Logger`
/// installed via `slog-stdlog::set_logger` would log things to a `StdLog`
/// drain, which would again log things to the global `Logger` and so on
/// leading to an infinite recursion.
pub struct StdLog;

struct LazyLogString<'a> {
    info: &'a slog::Record<'a>,
    logger_values: &'a slog::OwnedKVList,
}

impl<'a> LazyLogString<'a> {
    fn new(info: &'a slog::Record, logger_values: &'a slog::OwnedKVList) -> Self {
        LazyLogString {
            info,
            logger_values,
        }
    }
}

impl<'a> fmt::Display for LazyLogString<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.info.msg())?;

        let io = io::Cursor::new(Vec::new());
        let mut ser = KSV::new(io);

        self.logger_values
            .serialize(self.info, &mut ser)
            .map_err(|_| fmt::Error)?;
        self.info
            .kv()
            .serialize(self.info, &mut ser)
            .map_err(|_| fmt::Error)?;

        let values = ser.into_inner().into_inner();

        write!(f, "{}", String::from_utf8_lossy(&values))
    }
}

impl slog::Drain for StdLog {
    type Ok = ();
    type Err = io::Error;

    fn log(&self, info: &slog::Record, logger_values: &slog::OwnedKVList) -> io::Result<()> {
        let level = match info.level() {
            slog::Level::Critical | slog::Level::Error => log::Level::Error,
            slog::Level::Warning => log::Level::Warn,
            slog::Level::Info => log::Level::Info,
            slog::Level::Debug => log::Level::Debug,
            slog::Level::Trace => log::Level::Trace,
        };

        let mut target = info.tag();
        if target.is_empty() {
            target = info.module();
        }

        let lazy = LazyLogString::new(info, logger_values);
        /*
         * TODO: Support `log` crate key_values here.
         *
         * This requires the log/kv_unstable feature here.
         *
         * Not supporting this feature is backwards compatible
         * and it shouldn't break anything (because we've never had),
         * but is undesirable from a feature-completeness point of view.
         *
         * However, this is most likely not as powerful as slog's own
         * notion of key/value pairs, so I would humbly suggest using `slog`
         * directly if this feature is important to you ;)
         *
         * This avoids using the private log::__private_api_log api function,
         * which is just a thin wrapper around a `RecordBuilder`.
         */
        log::logger().log(
            &log::Record::builder()
                .args(format_args!("{}", lazy))
                .level(level)
                .target(target)
                .module_path_static(Some(info.module()))
                .file_static(Some(info.file()))
                .line(Some(info.line()))
                .build(),
        );

        Ok(())
    }
}

/// Key-Separator-Value serializer
struct KSV<W: io::Write> {
    io: W,
}

impl<W: io::Write> KSV<W> {
    fn new(io: W) -> Self {
        KSV { io: io }
    }

    fn into_inner(self) -> W {
        self.io
    }
}

impl<W: io::Write> slog::Serializer for KSV<W> {
    fn emit_arguments(&mut self, key: slog::Key, val: &fmt::Arguments) -> slog::Result {
        write!(self.io, ", {}: {}", key, val)?;
        Ok(())
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#[crate_id = "github.com/mozilla-servo/rust-core-text#core_text:0.1"];

extern crate std;

extern crate core_foundation;
extern crate core_graphics;

pub mod font;
pub mod font_collection;
pub mod font_descriptor;
pub mod font_manager;
<commit_msg>Generate both dylib and rlib for rustup<commit_after>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#[crate_id = "github.com/mozilla-servo/rust-core-text#core_text:0.1"];
#[crate_type = "lib"];
#[crate_type = "dylib"];
#[crate_type = "rlib"];

extern crate std;

extern crate core_foundation;
extern crate core_graphics;

pub mod font;
pub mod font_collection;
pub mod font_descriptor;
pub mod font_manager;
<|endoftext|>


<commit_before>/*!

LZ4 Decompression

This module contains an implementation in Rust of decompression of LZ4-encoded
streams. This is exposed as a standard `Reader` interface wrapping an
underlying `Reader` interface. In this manner, the decompressor can be viewed
as a stream. It maintains its own internal buffer, but the size of this is
constant as the stream is read.

# Example

```rust
let stream = Path::new("path/to/file.lz4").open_reader(io::Open);
let decompressed = l4z::Decoder::new(stream).read_to_end();
```

# Credit

This implementation is largely based on Branimir Karadžić's implementation which
can be found at https://github.com/bkaradzic/go-lz4.

*/

use std::rt::io;
use std::rt::io::extensions::{ReaderUtil, ReaderByteConversions};
use std::vec;
use std::num;

#[deriving(Eq)]
enum State {
    StreamStart,
    BlockStart,
    BlockEnd,

    // raw blocks of uncompressed data
    RawBlock(u32),

    // blocks of compressed data (no headers or checksums)
    DataStart,
    NeedsConsumption(u8, uint),
    DecompressStart(u8),
    Decompress(uint),

    EndOfStream,
    Cleanup,
}

enum ConsumptionResult {
    KeepGoing(uint),
    Consumed(uint),
    SomethingBad,
}

static BUF_SIZE: uint = 128 << 10;
static FLUSH_SIZE: uint = 1 << 16;
static MAGIC: u32 = 0x184d2204;

/// A decoder of an underlying lz4 stream. This is the decompressor used to
/// inflate an underlying stream.
pub struct Decoder<R> {
    /// This is the stream which is owned by this decoder. It may be moved out
    /// of, but the operation would also invalidate the `Decoder` instance.
    r: R,

    priv state: State,
    priv end: uint,
    priv start: uint,
    priv buf: ~[u8],
    priv remaining_bytes: uint,

    // metadata from the stream descriptor
    priv blk_checksum: bool,
    priv stream_checksum: bool,
}

impl<R: io::Reader + io::Seek> Decoder<R> {
    /// Creates a new decoder which will decompress the LZ4-encoded stream
    /// which will be read from `r`. This decoder will consume ownership of the
    /// reader, but it is accessible via the `r` field on the object returned.
    pub fn new(r: R) -> Decoder<R> {
        Decoder {
            r: r,
            state: StreamStart,
            start: 0,
            end: 0,
            buf: vec::from_elem(BUF_SIZE, 0u8),
            blk_checksum: false,
            stream_checksum: false,
            remaining_bytes: 0,
        }
    }

    fn inner_byte(&mut self) -> Option<u8> {
        match self.r.read_byte() {
            Some(b) => {
                self.remaining_bytes -= 1;
                Some(b)
            }
            None => None,
        }
    }

    fn inner_u16(&mut self) -> u16 {
        self.remaining_bytes -= 2;
        self.r.read_le_u16_()
    }

    fn inner_read(&mut self, dst: &mut [u8]) -> Option<uint> {
        match self.r.read(dst) {
            Some(n) => {
                self.remaining_bytes -= n;
                Some(n)
            }
            None => None,
        }
    }

    fn read_len(&mut self, code: u8) -> Option<uint> {
        let mut ret = code as uint;
        assert!(code < 16);
        if code == 0xf {
            loop {
                match self.inner_byte() {
                    None => return None,
                    Some(255) => ret += 255,
                    Some(n) => {
                        ret += n as uint;
                        break
                    }
                }
            }
        }
        return Some(ret);
    }

    fn consume(&mut self, len: uint, dst: &mut [u8]) -> ConsumptionResult {
        let in_output = self.flush(len, dst);
        if self.end + len > self.buf.len() {
            return KeepGoing(in_output);
        }

        let buf = unsafe {
            use std::cast;
            cast::transmute_copy(&self.buf.mut_slice(self.end, self.end + len))
        };
        match self.inner_read(buf) {
            Some(n) => {
                self.end += n;
                Consumed(in_output)
            }
            None => SomethingBad
        }
    }

    fn flush(&mut self, len: uint, dst: &mut [u8]) -> uint {
        if self.end + len > self.buf.len() {
            assert!(self.start > FLUSH_SIZE);
            let s = self.start - FLUSH_SIZE;

            // Copy bytes out into the destination
            let fill = num::min(dst.len(), s);
            info!("flushing out {}", fill);
            vec::bytes::copy_memory(dst, self.buf, fill);

            // Slide all the bytes back down in the internal buffer
            for i in range(0, fill) {
                self.buf[i] = self.buf[fill + i];
            }

            self.end -= fill;
            self.start -= fill;
            return fill;
        }
        return 0;
    }

    fn cp(&mut self, len: uint, decr: uint, dst: &mut [u8]) -> ConsumptionResult {
        let in_output = self.flush(len, dst);
        if self.end + len > self.buf.len() {
            return KeepGoing(in_output);
        }

        for i in range(0, len) {
            self.buf[self.end + i] = self.buf[self.start + i];
        }

        self.end += len;
        self.start += len - decr;
        return Consumed(in_output);
    }

    fn exec(&mut self, mut state: State, buf: &mut [u8]) -> Option<uint> {
        let mut offset = 0;
        loop {
            debug!("state: {:?} {:x} {:x} at {}", state, self.start, self.end,
                   self.r.tell());
            match state {
                StreamStart => {
                    match self.header() {
                        Some(()) => {}
                        None => return None,
                    }
                    state = BlockStart;
                }

                BlockStart => {
                    state = match self.r.read_le_u32_() {
                        0 => EndOfStream,
                        n if n & 0x80000000 != 0 => RawBlock(n & 0x7fffffff),
                        n => {
                            self.remaining_bytes = n as uint;
                            info!("bytes left {}", self.remaining_bytes);
                            DataStart
                        }
                    }
                }

                DataStart => {
                    //info!("remaining: {}", self.remaining_bytes as int);
                    assert!(self.remaining_bytes as int > 0);
                    let code = match self.inner_byte() {
                        Some(i) => i,
                        None => return None,
                    };
                    debug!("at: {:x} {} {:x}", self.r.tell(),
                           self.remaining_bytes as int, code);

                    // XXX: I/O errors
                    let len = match self.read_len(code >> 4) {
                        Some(l) => l, None => return None,
                    };
                    state = NeedsConsumption(code, len);
                }

                NeedsConsumption(code, len) => {
                    match self.consume(len, buf.mut_slice_from(offset)) {
                        // XXX: I/O error
                        SomethingBad => return None,
                        KeepGoing(amt_written) => {
                            self.state = state;
                            return Some(amt_written + offset);
                        }
                        Consumed(amt_written) => {
                            offset += amt_written;
                            if self.remaining_bytes == 0 {
                                state = BlockStart;
                            } else {
                                state = DecompressStart(code);
                            }
                        }
                    }
                }

                DecompressStart(code) => {
                    // XXX: I/O errors
                    let back = self.inner_u16();
                    debug!("got back: {}", back);
                    self.start = self.end - back as uint;

                    let len = match self.read_len(code & 0xf) {
                        Some(l) => l, None => return None,
                    };

                    state = Decompress(len);
                }

                Decompress(len) => {
                    let literal = self.end - self.start;
                    if literal < 4 {
                        static DECR: [uint, ..4] = [0, 3, 2, 3];
                        match self.cp(4, DECR[literal],
                                      buf.mut_slice_from(offset)) {
                            SomethingBad => unreachable!(),
                            KeepGoing(amt_written) => {
                                self.state = state;
                                return Some(amt_written + offset);
                            }
                            Consumed(amt_written) => offset += amt_written,
                        }
                    } else {
                        len += 4;
                    }
                    match self.cp(len, 0, buf.mut_slice_from(offset)) {
                        SomethingBad => unreachable!(),
                        KeepGoing(amt_written) => {
                            self.state = state;
                            return Some(amt_written + offset);
                        }
                        Consumed(amt_written) => {
                            offset += amt_written;
                            state = DataStart;
                        }
                    }
                }

                RawBlock(remaining) => {
                    let dst = buf.mut_slice_from(offset);
                    if dst.len() == 0 {
                        self.state = state;
                        return Some(offset);
                    }
                    if remaining == 0 {
                        state = BlockEnd;
                        continue
                    }
                    let amt = num::min(remaining as uint, dst.len());
                    match self.r.read(dst.mut_slice(offset, offset + amt)) {
                        Some(n) => {
                            offset += n;
                            state = RawBlock(remaining - n as u32);
                        }
                        None => return None
                    }
                }

                BlockEnd => {
                    // XXX: implement checksums
                    if self.blk_checksum {
                        let cksum = self.r.read_le_u32_();
                        debug!("ignoring block cksum: {:?}", cksum);
                    }
                    state = BlockStart;
                }

                EndOfStream => {
                    // XXX: implement checksums
                    if self.stream_checksum {
                        let cksum = self.r.read_le_u32_();
                        debug!("ignoring stream checksum: {:?}", cksum);
                    }
                    state = Cleanup;
                    self.start = 0;
                }

                Cleanup => {
                    let dst = buf.mut_slice_from(offset);
                    let src = self.buf.slice(self.start, self.end);
                    let amt = num::min(dst.len(), src.len());
                    if amt == 0 { return None }
                    vec::bytes::copy_memory(dst, src, amt);
                    self.start += amt;
                    self.state = Cleanup;
                    return Some(amt + offset);
                }
            }
        }
    }

    fn header(&mut self) -> Option<()> {
        // Make sure the magic number is what's expected.
        if self.r.read_le_u32_() != MAGIC { return None }

        let flg = match self.r.read_byte() { Some(i) => i, None => return None };
        let bd = match self.r.read_byte() { Some(i) => i, None => return None };

        // bits 7/6, the version number. Right now this must be 1
        if (flg >> 6) != 0b01 { return None }
        // bit 5 is the "block independence", don't care about this yet
        // bit 4 is whether blocks have checksums or not
        self.blk_checksum = (flg & 0x10) != 0;
        // bit 3 is whether there is a following stream size
        let stream_size = (flg & 0x08) != 0;
        // bit 2 is whether there is a stream checksum
        self.stream_checksum = (flg & 0x04) != 0;
        // bit 1 is reserved
        // bit 0 is whether there is a preset dictionary
        let preset_dictionary = (flg & 0x01) != 0;

        static MAX_SIZES: [uint, ..8] =
            [0, 0, 0, 0, // all N/A
             64 << 10,   // 64KB
             256 << 10,  // 256 KB
             1 << 20,    // 1MB
             4 << 20];   // 4MB

        // bit 7 is reserved
        // bits 6-4 are the maximum block size
        let max_block_size = MAX_SIZES[(bd >> 4) & 0x7];
        // bits 3-0 are reserved

        // read off other portions of the stream
        let size = if stream_size {Some(self.r.read_le_u64_())} else {None};
        assert!(!preset_dictionary, "preset dictionaries not supported yet");

        debug!("blk: {}", self.blk_checksum);
        debug!("stream: {}", self.stream_checksum);
        debug!("max size: {}", max_block_size);
        debug!("stream size: {:?}", size);

        // XXX: implement checksums
        let cksum = self.r.read_byte();
        debug!("ignoring header checksum: {:?}", cksum);
        return Some(());
    }
}

impl<R: io::Reader + io::Seek> io::Reader for Decoder<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        info!("reading {}", buf.len());
        let out = self.exec(self.state, buf);
        info!("got {:?}", out);
        out
    }

    fn eof(&mut self) -> bool {
        self.state == Cleanup && self.start == self.end
    }
}

#[cfg(test)]
mod test {
    use super::Decoder;
    use std::rt::io::extensions::ReaderUtil;
    use std::rt::io::mem::BufReader;
    use std::rt::io::Reader;
    use std::rand;

    fn test_decode(input: &[u8], output: &[u8]) {
        let mut d = Decoder::new(BufReader::new(input));

        let got = d.read_to_end();
        assert!(got.as_slice() == output);
    }

    #[test]
    fn decode() {
        let reference = include_bin!("data/test.txt");
        test_decode(include_bin!("data/test.lz4.1"), reference);
        test_decode(include_bin!("data/test.lz4.2"), reference);
        test_decode(include_bin!("data/test.lz4.3"), reference);
        test_decode(include_bin!("data/test.lz4.4"), reference);
        test_decode(include_bin!("data/test.lz4.5"), reference);
        test_decode(include_bin!("data/test.lz4.6"), reference);
        test_decode(include_bin!("data/test.lz4.7"), reference);
        test_decode(include_bin!("data/test.lz4.8"), reference);
        test_decode(include_bin!("data/test.lz4.9"), reference);
    }

    #[test]
    fn one_byte_at_a_time() {
        let input = include_bin!("data/test.lz4.1");
        let mut d = Decoder::new(BufReader::new(input));
        assert!(!d.eof());
        let mut out = ~[];
        loop {
            match d.read_byte() {
                Some(b) => out.push(b),
                None => break
            }
        }
        assert!(d.eof());
        assert!(out.as_slice() == include_bin!("data/test.txt"));
    }

    #[test]
    fn random_byte_lengths() {
        let input = include_bin!("data/test.lz4.1");
        let mut d = Decoder::new(BufReader::new(input));
        let mut out = ~[];
        let mut buf = [0u8, ..40];
        loop {
            match d.read(buf.mut_slice_to(1 + rand::random::<uint>() % 40)) {
                Some(n) => {
                    out.push_all(buf.slice_to(n));
                }
                None => break
            }
        }
        assert!(out.as_slice() == include_bin!("data/test.txt"));
    }
}
<commit_msg>Drop the Seek bound used in debugging<commit_after>/*!

LZ4 Decompression

This module contains an implementation in Rust of decompression of LZ4-encoded
streams. This is exposed as a standard `Reader` interface wrapping an
underlying `Reader` interface. In this manner, the decompressor can be viewed
as a stream. It maintains its own internal buffer, but the size of this is
constant as the stream is read.

# Example

```rust
let stream = Path::new("path/to/file.lz4").open_reader(io::Open);
let decompressed = l4z::Decoder::new(stream).read_to_end();
```

# Credit

This implementation is largely based on Branimir Karadžić's implementation which
can be found at https://github.com/bkaradzic/go-lz4.

*/

use std::rt::io;
use std::rt::io::extensions::{ReaderUtil, ReaderByteConversions};
use std::vec;
use std::num;

#[deriving(Eq)]
enum State {
    StreamStart,
    BlockStart,
    BlockEnd,

    // raw blocks of uncompressed data
    RawBlock(u32),

    // blocks of compressed data (no headers or checksums)
    DataStart,
    NeedsConsumption(u8, uint),
    DecompressStart(u8),
    Decompress(uint),

    EndOfStream,
    Cleanup,
}

enum ConsumptionResult {
    KeepGoing(uint),
    Consumed(uint),
    SomethingBad,
}

static BUF_SIZE: uint = 128 << 10;
static FLUSH_SIZE: uint = 1 << 16;
static MAGIC: u32 = 0x184d2204;

/// A decoder of an underlying lz4 stream. This is the decompressor used to
/// inflate an underlying stream.
pub struct Decoder<R> {
    /// This is the stream which is owned by this decoder. It may be moved out
    /// of, but the operation would also invalidate the `Decoder` instance.
    r: R,

    priv state: State,
    priv end: uint,
    priv start: uint,
    priv buf: ~[u8],
    priv remaining_bytes: uint,

    // metadata from the stream descriptor
    priv blk_checksum: bool,
    priv stream_checksum: bool,
}

impl<R: io::Reader> Decoder<R> {
    /// Creates a new decoder which will decompress the LZ4-encoded stream
    /// which will be read from `r`. This decoder will consume ownership of the
    /// reader, but it is accessible via the `r` field on the object returned.
    pub fn new(r: R) -> Decoder<R> {
        Decoder {
            r: r,
            state: StreamStart,
            start: 0,
            end: 0,
            buf: vec::from_elem(BUF_SIZE, 0u8),
            blk_checksum: false,
            stream_checksum: false,
            remaining_bytes: 0,
        }
    }

    fn inner_byte(&mut self) -> Option<u8> {
        match self.r.read_byte() {
            Some(b) => {
                self.remaining_bytes -= 1;
                Some(b)
            }
            None => None,
        }
    }

    fn inner_u16(&mut self) -> u16 {
        self.remaining_bytes -= 2;
        self.r.read_le_u16_()
    }

    fn inner_read(&mut self, dst: &mut [u8]) -> Option<uint> {
        match self.r.read(dst) {
            Some(n) => {
                self.remaining_bytes -= n;
                Some(n)
            }
            None => None,
        }
    }

    fn read_len(&mut self, code: u8) -> Option<uint> {
        let mut ret = code as uint;
        assert!(code < 16);
        if code == 0xf {
            loop {
                match self.inner_byte() {
                    None => return None,
                    Some(255) => ret += 255,
                    Some(n) => {
                        ret += n as uint;
                        break
                    }
                }
            }
        }
        return Some(ret);
    }

    fn consume(&mut self, len: uint, dst: &mut [u8]) -> ConsumptionResult {
        let in_output = self.flush(len, dst);
        if self.end + len > self.buf.len() {
            return KeepGoing(in_output);
        }

        let buf = unsafe {
            use std::cast;
            cast::transmute_copy(&self.buf.mut_slice(self.end, self.end + len))
        };
        match self.inner_read(buf) {
            Some(n) => {
                self.end += n;
                Consumed(in_output)
            }
            None => SomethingBad
        }
    }

    fn flush(&mut self, len: uint, dst: &mut [u8]) -> uint {
        if self.end + len > self.buf.len() {
            assert!(self.start > FLUSH_SIZE);
            let s = self.start - FLUSH_SIZE;

            // Copy bytes out into the destination
            let fill = num::min(dst.len(), s);
            info!("flushing out {}", fill);
            vec::bytes::copy_memory(dst, self.buf, fill);

            // Slide all the bytes back down in the internal buffer
            for i in range(0, fill) {
                self.buf[i] = self.buf[fill + i];
            }

            self.end -= fill;
            self.start -= fill;
            return fill;
        }
        return 0;
    }

    fn cp(&mut self, len: uint, decr: uint, dst: &mut [u8]) -> ConsumptionResult {
        let in_output = self.flush(len, dst);
        if self.end + len > self.buf.len() {
            return KeepGoing(in_output);
        }

        for i in range(0, len) {
            self.buf[self.end + i] = self.buf[self.start + i];
        }

        self.end += len;
        self.start += len - decr;
        return Consumed(in_output);
    }

    fn exec(&mut self, mut state: State, buf: &mut [u8]) -> Option<uint> {
        let mut offset = 0;
        loop {
            debug!("state: {:?} {:x} {:x}", state, self.start, self.end);
            match state {
                StreamStart => {
                    match self.header() {
                        Some(()) => {}
                        None => return None,
                    }
                    state = BlockStart;
                }

                BlockStart => {
                    state = match self.r.read_le_u32_() {
                        0 => EndOfStream,
                        n if n & 0x80000000 != 0 => RawBlock(n & 0x7fffffff),
                        n => {
                            self.remaining_bytes = n as uint;
                            info!("bytes left {}", self.remaining_bytes);
                            DataStart
                        }
                    }
                }

                DataStart => {
                    //info!("remaining: {}", self.remaining_bytes as int);
                    assert!(self.remaining_bytes as int > 0);
                    let code = match self.inner_byte() {
                        Some(i) => i,
                        None => return None,
                    };
                    debug!("at: {:x} {}", self.remaining_bytes as int, code);

                    // XXX: I/O errors
                    let len = match self.read_len(code >> 4) {
                        Some(l) => l, None => return None,
                    };
                    state = NeedsConsumption(code, len);
                }

                NeedsConsumption(code, len) => {
                    match self.consume(len, buf.mut_slice_from(offset)) {
                        // XXX: I/O error
                        SomethingBad => return None,
                        KeepGoing(amt_written) => {
                            self.state = state;
                            return Some(amt_written + offset);
                        }
                        Consumed(amt_written) => {
                            offset += amt_written;
                            if self.remaining_bytes == 0 {
                                state = BlockStart;
                            } else {
                                state = DecompressStart(code);
                            }
                        }
                    }
                }

                DecompressStart(code) => {
                    // XXX: I/O errors
                    let back = self.inner_u16();
                    debug!("got back: {}", back);
                    self.start = self.end - back as uint;

                    let len = match self.read_len(code & 0xf) {
                        Some(l) => l, None => return None,
                    };

                    state = Decompress(len);
                }

                Decompress(len) => {
                    let literal = self.end - self.start;
                    if literal < 4 {
                        static DECR: [uint, ..4] = [0, 3, 2, 3];
                        match self.cp(4, DECR[literal],
                                      buf.mut_slice_from(offset)) {
                            SomethingBad => unreachable!(),
                            KeepGoing(amt_written) => {
                                self.state = state;
                                return Some(amt_written + offset);
                            }
                            Consumed(amt_written) => offset += amt_written,
                        }
                    } else {
                        len += 4;
                    }
                    match self.cp(len, 0, buf.mut_slice_from(offset)) {
                        SomethingBad => unreachable!(),
                        KeepGoing(amt_written) => {
                            self.state = state;
                            return Some(amt_written + offset);
                        }
                        Consumed(amt_written) => {
                            offset += amt_written;
                            state = DataStart;
                        }
                    }
                }

                RawBlock(remaining) => {
                    let dst = buf.mut_slice_from(offset);
                    if dst.len() == 0 {
                        self.state = state;
                        return Some(offset);
                    }
                    if remaining == 0 {
                        state = BlockEnd;
                        continue
                    }
                    let amt = num::min(remaining as uint, dst.len());
                    match self.r.read(dst.mut_slice(offset, offset + amt)) {
                        Some(n) => {
                            offset += n;
                            state = RawBlock(remaining - n as u32);
                        }
                        None => return None
                    }
                }

                BlockEnd => {
                    // XXX: implement checksums
                    if self.blk_checksum {
                        let cksum = self.r.read_le_u32_();
                        debug!("ignoring block cksum: {:?}", cksum);
                    }
                    state = BlockStart;
                }

                EndOfStream => {
                    // XXX: implement checksums
                    if self.stream_checksum {
                        let cksum = self.r.read_le_u32_();
                        debug!("ignoring stream checksum: {:?}", cksum);
                    }
                    state = Cleanup;
                    self.start = 0;
                }

                Cleanup => {
                    let dst = buf.mut_slice_from(offset);
                    let src = self.buf.slice(self.start, self.end);
                    let amt = num::min(dst.len(), src.len());
                    if amt == 0 { return None }
                    vec::bytes::copy_memory(dst, src, amt);
                    self.start += amt;
                    self.state = Cleanup;
                    return Some(amt + offset);
                }
            }
        }
    }

    fn header(&mut self) -> Option<()> {
        // Make sure the magic number is what's expected.
        if self.r.read_le_u32_() != MAGIC { return None }

        let flg = match self.r.read_byte() { Some(i) => i, None => return None };
        let bd = match self.r.read_byte() { Some(i) => i, None => return None };

        // bits 7/6, the version number. Right now this must be 1
        if (flg >> 6) != 0b01 { return None }
        // bit 5 is the "block independence", don't care about this yet
        // bit 4 is whether blocks have checksums or not
        self.blk_checksum = (flg & 0x10) != 0;
        // bit 3 is whether there is a following stream size
        let stream_size = (flg & 0x08) != 0;
        // bit 2 is whether there is a stream checksum
        self.stream_checksum = (flg & 0x04) != 0;
        // bit 1 is reserved
        // bit 0 is whether there is a preset dictionary
        let preset_dictionary = (flg & 0x01) != 0;

        static MAX_SIZES: [uint, ..8] =
            [0, 0, 0, 0, // all N/A
             64 << 10,   // 64KB
             256 << 10,  // 256 KB
             1 << 20,    // 1MB
             4 << 20];   // 4MB

        // bit 7 is reserved
        // bits 6-4 are the maximum block size
        let max_block_size = MAX_SIZES[(bd >> 4) & 0x7];
        // bits 3-0 are reserved

        // read off other portions of the stream
        let size = if stream_size {Some(self.r.read_le_u64_())} else {None};
        assert!(!preset_dictionary, "preset dictionaries not supported yet");

        debug!("blk: {}", self.blk_checksum);
        debug!("stream: {}", self.stream_checksum);
        debug!("max size: {}", max_block_size);
        debug!("stream size: {:?}", size);

        // XXX: implement checksums
        let cksum = self.r.read_byte();
        debug!("ignoring header checksum: {:?}", cksum);
        return Some(());
    }
}

impl<R: io::Reader> io::Reader for Decoder<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        info!("reading {}", buf.len());
        let out = self.exec(self.state, buf);
        info!("got {:?}", out);
        out
    }

    fn eof(&mut self) -> bool {
        self.state == Cleanup && self.start == self.end
    }
}

#[cfg(test)]
mod test {
    use super::Decoder;
    use std::rt::io::extensions::ReaderUtil;
    use std::rt::io::mem::BufReader;
    use std::rt::io::Reader;
    use std::rand;

    fn test_decode(input: &[u8], output: &[u8]) {
        let mut d = Decoder::new(BufReader::new(input));

        let got = d.read_to_end();
        assert!(got.as_slice() == output);
    }

    #[test]
    fn decode() {
        let reference = include_bin!("data/test.txt");
        test_decode(include_bin!("data/test.lz4.1"), reference);
        test_decode(include_bin!("data/test.lz4.2"), reference);
        test_decode(include_bin!("data/test.lz4.3"), reference);
        test_decode(include_bin!("data/test.lz4.4"), reference);
        test_decode(include_bin!("data/test.lz4.5"), reference);
        test_decode(include_bin!("data/test.lz4.6"), reference);
        test_decode(include_bin!("data/test.lz4.7"), reference);
        test_decode(include_bin!("data/test.lz4.8"), reference);
        test_decode(include_bin!("data/test.lz4.9"), reference);
    }

    #[test]
    fn one_byte_at_a_time() {
        let input = include_bin!("data/test.lz4.1");
        let mut d = Decoder::new(BufReader::new(input));
        assert!(!d.eof());
        let mut out = ~[];
        loop {
            match d.read_byte() {
                Some(b) => out.push(b),
                None => break
            }
        }
        assert!(d.eof());
        assert!(out.as_slice() == include_bin!("data/test.txt"));
    }

    #[test]
    fn random_byte_lengths() {
        let input = include_bin!("data/test.lz4.1");
        let mut d = Decoder::new(BufReader::new(input));
        let mut out = ~[];
        let mut buf = [0u8, ..40];
        loop {
            match d.read(buf.mut_slice_to(1 + rand::random::<uint>() % 40)) {
                Some(n) => {
                    out.push_all(buf.slice_to(n));
                }
                None => break
            }
        }
        assert!(out.as_slice() == include_bin!("data/test.txt"));
    }
}
<|endoftext|>