<commit_before>// -*- Mode: C++; tab-width: 2; -*-
// vi: set ts=2:
//
// $Id: ringPerceptionProcessor.C,v 1.3 2004/09/02 13:12:51 amoll Exp $
//

#include <BALL/QSAR/ringPerceptionProcessor.h>

#include <BALL/KERNEL/bond.h>
#include <BALL/KERNEL/fragment.h>
#include <BALL/KERNEL/bondIterator.h>
#include <BALL/KERNEL/atomIterator.h>
#include <BALL/KERNEL/forEach.h>
#include <BALL/KERNEL/PTE.h>
#include <BALL/COMMON/limits.h>

#include <iostream>
using std::endl;
#include <vector>
using std::vector;
#include <utility>
#include <queue>
using std::priority_queue;
using std::deque;
using std::pair;

namespace BALL
{
	RingPerceptionProcessor::RingPerceptionProcessor()
		:	UnaryProcessor<AtomContainer>()
	{
	}

	RingPerceptionProcessor::RingPerceptionProcessor(const RingPerceptionProcessor& rp)
		:	UnaryProcessor<AtomContainer>(rp)
	{
	}

	RingPerceptionProcessor& RingPerceptionProcessor::operator = (const RingPerceptionProcessor& /* rp */)
	{
		return *this;
	}

	RingPerceptionProcessor::~RingPerceptionProcessor()
	{
	}

	Processor::Result RingPerceptionProcessor::operator () (AtomContainer& AtomContainer)
	{
		vector<vector<Atom*> > sssr;
		calculateSSSR(sssr, AtomContainer);
		return Processor::CONTINUE;
	}

	Size RingPerceptionProcessor::calculateSSSR(vector<vector<Atom*> >& sssr_orig, AtomContainer& ac)
	{
		//cerr << "RingPerceptionProcessr::calculateSSSR(...)";
		AtomContainer AtomContainer;
		AtomContainer = ac; // the algorithm runs on a copy, bc bonds and maybe atoms are destroyed!

		// mapping is needed because this algorithms works on a copy
		HashMap<Atom*, Atom*> copy_to_orig;
		AtomIterator orig = ac.beginAtom();
		AtomIterator copy = AtomContainer.beginAtom();
		for (;orig!=ac.endAtom();++orig,++copy)
		{
			copy_to_orig.insert(std::make_pair(&(*copy),&(*orig)));
		}

		HashSet<Atom*> full_set; // herein are all nodes (atoms)
		HashSet<Atom*> trim_set; // herein are all the zero edge nodes
		vector<HashSet<Atom*> > SSSR; // stores the rings

		AtomIterator atom_it = AtomContainer.beginAtom();
		for (;atom_it!=AtomContainer.endAtom();++atom_it)
		{
			full_set.insert(&(*atom_it));
		}

		while (full_set.size() != trim_set.size())
		{
			HashSet<Atom*>::Iterator it = full_set.begin();
			unsigned int min_deg = Limits<int>::max();
			Atom * init;
			HashSet<Atom*> nodes_n2_set;
			for (;it!=full_set.end();++it)
			{
				//add all nodes with degree 0 to trim_set
				if ((*it)->countBonds() == 0)
				{
					trim_set.insert(*it);
				}
				else
				{
					// find beginning node init in full_set-trim_set with minimal degree
					if ((*it)->countBonds() < min_deg)
					{
						min_deg = (*it)->countBonds();
						init = *it;
					}
					// add n2-nodes from full_set - trim_set to nodes_n2
					if ((*it)->countBonds() == 2)
					{
						nodes_n2_set.insert(*it);
					}
				}
			}

			if (min_deg == 1)
			{
				init->destroyBond(*(init->beginBond()->getPartner(*init)));
				trim_set.insert(init);
			}
			else
			{
				HashSet<Atom*> ring_set;
				if (min_deg == 2)
				{
					for (it=nodes_n2_set.begin();it!=nodes_n2_set.end();++it)
					{
						Size ring_size = getRing_(*it, ring_set);
						if (ring_size > 2)
						{
							// add ring_set to SSSR
							SSSR.push_back(ring_set);
						}
					}
					// for every chain of n2 nodes, isolate one and delete the chain it belongs
					HashSet<Atom*>::Iterator it2;
					for (it2=nodes_n2_set.begin();it2!=nodes_n2_set.end();++it2)
					{
						if ((*it2)->countBonds() != 0)
						{
							if (!(*it2)->destroyBond(*((*it2)->beginBond()->getPartner(**it2))))
							{
								// should really not occur!
							}
						}
					}
				}
				else
				{
					if (min_deg >= 3)
					{
						Size ring_size = getRing_(init, ring_set);
						if (ring_size > 0)
						{
							// add ring_set ro sssr
							SSSR.push_back(ring_set);
							checkEdges_(ring_set, AtomContainer);
						}
					}
				}
			}
		}

		// now put the computet rings in the referenced structure
		for (vector<HashSet<Atom*> >::iterator i=SSSR.begin();i!=SSSR.end();++i)
		{
			vector<Atom*> ring;
			for (HashSet<Atom*>::Iterator j=i->begin();j!=i->end();++j)
			{
				ring.push_back(copy_to_orig[*j]);
			}
			sort(ring.begin(),ring.end());
			sssr_orig.push_back(ring);
		}

		// erase the copies (erasing here should be faster than searching by every input
		sort(sssr_orig.begin(),sssr_orig.end());
		vector<vector<Atom*> >::iterator vhs_it = unique(sssr_orig.begin(),sssr_orig.end());
		sssr_orig.erase(vhs_it,sssr_orig.end());

		// set "InRing" properties of the bond and atoms;
    // save the atoms as "InRing" as atom property
    for (Size i=0;i!=sssr_orig.size();++i)
    {
			for (Size j=0;j!=sssr_orig[i].size();++j)
			{
				sssr_orig[i][j]->setProperty("InRing", true);
			}
			for (Size j=0;j!=sssr_orig[i].size();++j)
			{
				for (Size k=0;k!=sssr_orig[i].size();++k)
				{
					if (sssr_orig[i][k]->isBoundTo(*sssr_orig[i][j]))
					{
						sssr_orig[i][k]->getBond(*sssr_orig[i][j])->setProperty("InRing", true);
					}
				}
			}
		}

		// the atoms that are not participating a ring
   	for (AtomIterator i=AtomContainer.beginAtom();i!=AtomContainer.endAtom();++i)
		{
		  if (!i->hasProperty("InRing"))
		  {
				i->setProperty("InRing", false);
			}
		}

		AtomIterator a_it = AtomContainer.beginAtom();
		Atom::BondIterator b_it = a_it->beginBond();
		BALL_FOREACH_BOND (AtomContainer, a_it, b_it)
		{
			if (!b_it->hasProperty("InRing"))
			{
				b_it->setProperty("InRing", false);
			}
		}

		//cerr << "..ended!" << endl;

		return sssr_orig.size();
	}

	Size RingPerceptionProcessor::getRing_(Atom* n, HashSet<Atom*>& ring_set)
	{
		deque<std::pair<Atom*, Atom*> > the_q; // double ended queue with node and its ancestor
		the_q.push_back(std::make_pair(n,n));

		Atom::BondIterator bond_it;
		HashMap<Atom*, HashSet<Atom*> > paths;
		HashSet<Atom*> tmp;
		tmp.insert(n);
		paths.insert(std::make_pair(n, tmp));

		while (!the_q.empty())
		{
			pair<Atom*, Atom*> atom_anc = the_q.front();
			Atom* atom = atom_anc.first;
			Atom* ancestor = atom_anc.second;
			the_q.pop_front();
			for (bond_it=atom->beginBond();bond_it!=atom->endBond();++bond_it)
			{
				Atom* bound_atom =  bond_it->getPartner(*atom);
				if (bound_atom != ancestor)
				{
					if (!paths.has(bound_atom))
					{
						HashSet<Atom*> path_atom = paths[atom];
						path_atom.insert(bound_atom);
						paths.insert(std::make_pair(bound_atom, path_atom));
					}
					else
					{
						HashSet<Atom*> merge, atom1, atom2;
						atom1 = paths[bound_atom];
						atom2 = paths[atom];
						merge.set(atom1);

						// workaround, don't know how to merge to hashsets, the sets are small,
						// doesn't touch the performance that much
						HashSet<Atom*>::Iterator set_it;
						for (set_it=atom2.begin();set_it!=atom2.end();++set_it)
						{
							if (!merge.has(*set_it))
							{
								merge.insert(*set_it);
							}
						}

						if (atom1.size()+atom2.size()-merge.size() == 1)
						{
							ring_set = merge;
							return merge.size();
						}
					}
					the_q.push_back(std::make_pair(bound_atom, atom));
				}
			}
		}
		return 0;
	}

	void RingPerceptionProcessor::checkEdges_(HashSet<Atom*>& ring_set, AtomContainer& ac)
	{
		// Every node with degree 3 (which is a memeber of ring_set) is deleted testwise.
		// Finally the node which produces the smalles ring is deleted.

		HashSet<Atom*>::Iterator iter=ring_set.begin();
		Atom * min_atom = *iter; // set as default to avoid compiler warning
		Size min_ring_size = Limits<int>::max();

		for (;iter!=ring_set.end();++iter)
		{
			if ((*iter)->countBonds() == 3)
			{
				AtomContainer AtomContainer = ac;

				// mapping to switch between copy and orig AtomContainer
				HashMap<Atom*, Atom*> copy_to_orig;
				HashMap<Atom*, Atom*> orig_to_copy;
				AtomIterator copy = AtomContainer.beginAtom();
				AtomIterator orig = ac.beginAtom();
				for (;orig!=ac.endAtom();++copy, ++orig)
				{
					copy_to_orig.insert(std::make_pair(&(*copy),&(*orig)));
					orig_to_copy.insert(std::make_pair(&(*orig),&(*copy)));
				}

				orig_to_copy[*iter]->destroy();

				Atom::BondIterator bond_it=(*iter)->beginBond();
				for (;bond_it!=(*iter)->endBond();++bond_it)
				{
					Atom* atom = bond_it->getPartner(**iter);
					HashSet<Atom*> rs;
					Size size = getRing_(atom, rs);

					if (min_ring_size > size)
					{
						min_ring_size = size;
						min_atom = *iter;
					}
				}
			}
		}
		// now delete min_atom
		min_atom->destroy();
	}

} // namespace BALL
<commit_msg>fixed a source of core dumps<commit_after>// -*- Mode: C++; tab-width: 2; -*-
// vi: set ts=2:
//
// $Id: ringPerceptionProcessor.C,v 1.4 2004/09/07 13:17:36 amoll Exp $
//

#include <BALL/QSAR/ringPerceptionProcessor.h>

#include <BALL/KERNEL/bond.h>
#include <BALL/KERNEL/fragment.h>
#include <BALL/KERNEL/bondIterator.h>
#include <BALL/KERNEL/atomIterator.h>
#include <BALL/KERNEL/forEach.h>
#include <BALL/KERNEL/PTE.h>
#include <BALL/COMMON/limits.h>

#include <iostream>
using std::endl;
#include <vector>
using std::vector;
#include <utility>
#include <queue>
using std::priority_queue;
using std::deque;
using std::pair;

namespace BALL
{
	RingPerceptionProcessor::RingPerceptionProcessor()
		:	UnaryProcessor<AtomContainer>()
	{
	}

	RingPerceptionProcessor::RingPerceptionProcessor(const RingPerceptionProcessor& rp)
		:	UnaryProcessor<AtomContainer>(rp)
	{
	}

	RingPerceptionProcessor& RingPerceptionProcessor::operator = (const RingPerceptionProcessor& /* rp */)
	{
		return *this;
	}

	RingPerceptionProcessor::~RingPerceptionProcessor()
	{
	}

	Processor::Result RingPerceptionProcessor::operator () (AtomContainer& AtomContainer)
	{
		vector<vector<Atom*> > sssr;
		calculateSSSR(sssr, AtomContainer);
		return Processor::CONTINUE;
	}

	Size RingPerceptionProcessor::calculateSSSR(vector<vector<Atom*> >& sssr_orig, AtomContainer& ac)
	{
		//cerr << "RingPerceptionProcessr::calculateSSSR(...)";
		AtomContainer AtomContainer;
		AtomContainer = ac; // the algorithm runs on a copy, bc bonds and maybe atoms are destroyed!

		// mapping is needed because this algorithms works on a copy
		HashMap<Atom*, Atom*> copy_to_orig;
		AtomIterator orig = ac.beginAtom();
		AtomIterator copy = AtomContainer.beginAtom();
		for (;orig!=ac.endAtom();++orig,++copy)
		{
			copy_to_orig.insert(std::make_pair(&(*copy),&(*orig)));
		}

		HashSet<Atom*> full_set; // herein are all nodes (atoms)
		HashSet<Atom*> trim_set; // herein are all the zero edge nodes
		vector<HashSet<Atom*> > SSSR; // stores the rings

		AtomIterator atom_it = AtomContainer.beginAtom();
		for (;atom_it!=AtomContainer.endAtom();++atom_it)
		{
			full_set.insert(&(*atom_it));
		}

		while (full_set.size() != trim_set.size())
		{
			HashSet<Atom*>::Iterator it = full_set.begin();
			unsigned int min_deg = Limits<int>::max();
			Atom * init;
			HashSet<Atom*> nodes_n2_set;
			for (;it!=full_set.end();++it)
			{
				//add all nodes with degree 0 to trim_set
				if ((*it)->countBonds() == 0)
				{
					trim_set.insert(*it);
				}
				else
				{
					// find beginning node init in full_set-trim_set with minimal degree
					if ((*it)->countBonds() < min_deg)
					{
						min_deg = (*it)->countBonds();
						init = *it;
					}
					// add n2-nodes from full_set - trim_set to nodes_n2
					if ((*it)->countBonds() == 2)
					{
						nodes_n2_set.insert(*it);
					}
				}
			}

			if (min_deg == 1)
			{
				init->destroyBond(*(init->beginBond()->getPartner(*init)));
				trim_set.insert(init);
			}
			else
			{
				HashSet<Atom*> ring_set;
				if (min_deg == 2)
				{
					for (it=nodes_n2_set.begin();it!=nodes_n2_set.end();++it)
					{
						Size ring_size = getRing_(*it, ring_set);
						if (ring_size > 2)
						{
							// add ring_set to SSSR
							SSSR.push_back(ring_set);
						}
					}
					// for every chain of n2 nodes, isolate one and delete the chain it belongs
					HashSet<Atom*>::Iterator it2;
					for (it2=nodes_n2_set.begin();it2!=nodes_n2_set.end();++it2)
					{
						if ((*it2)->countBonds() != 0)
						{
							if (!(*it2)->destroyBond(*((*it2)->beginBond()->getPartner(**it2))))
							{
								// should really not occur!
							}
						}
					}
				}
				else
				{
					if (min_deg >= 3)
					{
						Size ring_size = getRing_(init, ring_set);
						if (ring_size > 0)
						{
							// add ring_set ro sssr
							SSSR.push_back(ring_set);
							checkEdges_(ring_set, AtomContainer);
						}
					}
				}
			}
		}

		// now put the computet rings in the referenced structure
		for (vector<HashSet<Atom*> >::iterator i=SSSR.begin();i!=SSSR.end();++i)
		{
			vector<Atom*> ring;
			for (HashSet<Atom*>::Iterator j=i->begin();j!=i->end();++j)
			{
				ring.push_back(copy_to_orig[*j]);
			}
			sort(ring.begin(),ring.end());
			sssr_orig.push_back(ring);
		}

		// erase the copies (erasing here should be faster than searching by every input
		sort(sssr_orig.begin(),sssr_orig.end());
		vector<vector<Atom*> >::iterator vhs_it = unique(sssr_orig.begin(),sssr_orig.end());
		sssr_orig.erase(vhs_it,sssr_orig.end());

		// set "InRing" properties of the bond and atoms;
    // save the atoms as "InRing" as atom property
    for (Size i=0;i!=sssr_orig.size();++i)
    {
			for (Size j=0;j!=sssr_orig[i].size();++j)
			{
				sssr_orig[i][j]->setProperty("InRing", true);
			}
			for (Size j=0;j!=sssr_orig[i].size();++j)
			{
				for (Size k=0;k!=sssr_orig[i].size();++k)
				{
					if (sssr_orig[i][k]->isBoundTo(*sssr_orig[i][j]))
					{
						sssr_orig[i][k]->getBond(*sssr_orig[i][j])->setProperty("InRing", true);
					}
				}
			}
		}

		// the atoms that are not participating a ring
   	for (AtomIterator i=AtomContainer.beginAtom();i!=AtomContainer.endAtom();++i)
		{
		  if (!i->hasProperty("InRing"))
		  {
				i->setProperty("InRing", false);
			}
		}

		AtomIterator a_it = AtomContainer.beginAtom();
		Atom::BondIterator b_it = a_it->beginBond();
		BALL_FOREACH_BOND (AtomContainer, a_it, b_it)
		{
			if (!b_it->hasProperty("InRing"))
			{
				b_it->setProperty("InRing", false);
			}
		}

		//cerr << "..ended!" << endl;

		return sssr_orig.size();
	}

	Size RingPerceptionProcessor::getRing_(Atom* n, HashSet<Atom*>& ring_set)
	{
		deque<std::pair<Atom*, Atom*> > the_q; // double ended queue with node and its ancestor
		the_q.push_back(std::make_pair(n,n));

		Atom::BondIterator bond_it;
		HashMap<Atom*, HashSet<Atom*> > paths;
		HashSet<Atom*> tmp;
		tmp.insert(n);
		paths.insert(std::make_pair(n, tmp));

		while (!the_q.empty())
		{
			pair<Atom*, Atom*> atom_anc = the_q.front();
			Atom* atom = atom_anc.first;
			Atom* ancestor = atom_anc.second;
			the_q.pop_front();
			for (bond_it=atom->beginBond();bond_it!=atom->endBond();++bond_it)
			{
				Atom* bound_atom =  bond_it->getPartner(*atom);
				if (bound_atom != 0 &&
						bound_atom != ancestor)
				{
					if (!paths.has(bound_atom))
					{
						HashSet<Atom*> path_atom = paths[atom];
						path_atom.insert(bound_atom);
						paths.insert(std::make_pair(bound_atom, path_atom));
					}
					else
					{
						HashSet<Atom*> merge, atom1, atom2;
						atom1 = paths[bound_atom];
						atom2 = paths[atom];
						merge.set(atom1);

						// workaround, don't know how to merge to hashsets, the sets are small,
						// doesn't touch the performance that much
						HashSet<Atom*>::Iterator set_it;
						for (set_it=atom2.begin();set_it!=atom2.end();++set_it)
						{
							if (!merge.has(*set_it))
							{
								merge.insert(*set_it);
							}
						}

						if (atom1.size()+atom2.size()-merge.size() == 1)
						{
							ring_set = merge;
							return merge.size();
						}
					}
					the_q.push_back(std::make_pair(bound_atom, atom));
				}
			}
		}
		return 0;
	}

	void RingPerceptionProcessor::checkEdges_(HashSet<Atom*>& ring_set, AtomContainer& ac)
	{
		// Every node with degree 3 (which is a memeber of ring_set) is deleted testwise.
		// Finally the node which produces the smalles ring is deleted.

		HashSet<Atom*>::Iterator iter=ring_set.begin();
		Atom * min_atom = *iter; // set as default to avoid compiler warning
		Size min_ring_size = Limits<int>::max();

		for (;iter!=ring_set.end();++iter)
		{
			if ((*iter)->countBonds() == 3)
			{
				AtomContainer AtomContainer = ac;

				// mapping to switch between copy and orig AtomContainer
				HashMap<Atom*, Atom*> copy_to_orig;
				HashMap<Atom*, Atom*> orig_to_copy;
				AtomIterator copy = AtomContainer.beginAtom();
				AtomIterator orig = ac.beginAtom();
				for (;orig!=ac.endAtom();++copy, ++orig)
				{
					copy_to_orig.insert(std::make_pair(&(*copy),&(*orig)));
					orig_to_copy.insert(std::make_pair(&(*orig),&(*copy)));
				}

				orig_to_copy[*iter]->destroy();

				Atom::BondIterator bond_it=(*iter)->beginBond();
				for (;bond_it!=(*iter)->endBond();++bond_it)
				{
					Atom* atom = bond_it->getPartner(**iter);
					HashSet<Atom*> rs;
					Size size = getRing_(atom, rs);

					if (min_ring_size > size)
					{
						min_ring_size = size;
						min_atom = *iter;
					}
				}
			}
		}
		// now delete min_atom
		min_atom->destroy();
	}

} // namespace BALL
<|endoftext|>


<commit_before>//
//  yas_audio_engine_avf_au_mixer.cpp
//

#include "yas_audio_engine_avf_au_mixer.h"

using namespace yas;

audio::engine::avf_au_mixer::avf_au_mixer()
    : _au(avf_au::make_shared(
          {.acd =
               AudioComponentDescription{
                   .componentType = kAudioUnitType_Mixer,
                   .componentSubType = kAudioUnitSubType_MultiChannelMixer,
                   .componentManufacturer = kAudioUnitManufacturer_Apple,
                   .componentFlags = 0,
                   .componentFlagsMask = 0,
               },
           .node_args = {.input_bus_count = std::numeric_limits<uint32_t>::max(), .output_bus_count = 1}})) {
}

void audio::engine::avf_au_mixer::_prepare(avf_au_mixer_ptr const &shared) {
    /*
    this->_connections_observer = this->_au->chain(au::method::will_update_connections)
    .perform([weak_au_mixer = to_weak(shared)](auto const &) {
        if (auto au_mixer = weak_au_mixer.lock()) {
            au_mixer->_update_unit_mixer_connections();
        }
    })
    .end();
     */
}

void audio::engine::avf_au_mixer::_update_unit_mixer_connections() {
    auto const &connections = manageable_node::cast(this->_au->node())->input_connections();
    if (connections.size() > 0) {
        auto last = connections.end();
        --last;

        auto &pair = *last;
        this->_au->set_input_bus_count(pair.first + 1);
    }
}

audio::engine::avf_au_mixer_ptr audio::engine::avf_au_mixer::make_shared() {
    auto shared = std::shared_ptr<avf_au_mixer>(new avf_au_mixer{});
    return shared;
}
<commit_msg>observe connections<commit_after>//
//  yas_audio_engine_avf_au_mixer.cpp
//

#include "yas_audio_engine_avf_au_mixer.h"

using namespace yas;

audio::engine::avf_au_mixer::avf_au_mixer()
    : _au(avf_au::make_shared(
          {.acd =
               AudioComponentDescription{
                   .componentType = kAudioUnitType_Mixer,
                   .componentSubType = kAudioUnitSubType_MultiChannelMixer,
                   .componentManufacturer = kAudioUnitManufacturer_Apple,
                   .componentFlags = 0,
                   .componentFlagsMask = 0,
               },
           .node_args = {.input_bus_count = std::numeric_limits<uint32_t>::max(), .output_bus_count = 1}})) {
}

void audio::engine::avf_au_mixer::_prepare(avf_au_mixer_ptr const &shared) {
    this->_connections_observer =
        this->_au->connection_chain()
            .guard([](auto const &method) { return method == avf_au::connection_method::will_update; })
            .perform([weak_au_mixer = to_weak(shared)](auto const &) {
                if (auto au_mixer = weak_au_mixer.lock()) {
                    au_mixer->_update_unit_mixer_connections();
                }
            })
            .end();
}

void audio::engine::avf_au_mixer::_update_unit_mixer_connections() {
    auto const &connections = manageable_node::cast(this->_au->node())->input_connections();
    if (connections.size() > 0) {
        auto last = connections.end();
        --last;

        auto &pair = *last;
        this->_au->set_input_bus_count(pair.first + 1);
    }
}

audio::engine::avf_au_mixer_ptr audio::engine::avf_au_mixer::make_shared() {
    auto shared = std::shared_ptr<avf_au_mixer>(new avf_au_mixer{});
    return shared;
}
<|endoftext|>


<commit_before>#ifndef AMGCL_RELAXATION_DETAIL_ILU_SOLVE_HPP
#define AMGCL_RELAXATION_DETAIL_ILU_SOLVE_HPP

/*
The MIT License

Copyright (c) 2012-2017 Denis Demidov <dennis.demidov@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

/**
 * \file   amgcl/relaxation/detail/ilu_solve.hpp
 * \author Denis Demidov <dennis.demidov@gmail.com>
 * \brief  Solver for sparse triangular systems obtained as a result of an
 *         incomplete LU factorization.
 */

#include <amgcl/backend/interface.hpp>
#include <amgcl/util.hpp>

namespace amgcl {
namespace relaxation {
namespace detail {

template <class Backend>
class ilu_solve {
    public:
        typedef typename Backend::params backend_params;
        typedef typename Backend::value_type value_type;
        typedef typename Backend::matrix matrix;
        typedef typename Backend::vector vector;
        typedef typename Backend::matrix_diagonal matrix_diagonal;
        typedef typename backend::builtin<value_type>::matrix build_matrix;
        typedef typename math::scalar_of<value_type>::type scalar_type;

        struct params {
            /// Number of Jacobi iterations.
            unsigned    iters;

            /// Damping factor.
            scalar_type damping;

            params() : iters(2), damping(0.72) {}

            params(const boost::property_tree::ptree &p)
                : AMGCL_PARAMS_IMPORT_VALUE(p, iters)
                , AMGCL_PARAMS_IMPORT_VALUE(p, damping)
            {
                AMGCL_PARAMS_CHECK(p, (iters)(damping));
            }

            void get(boost::property_tree::ptree &p, const std::string &path) const {
                AMGCL_PARAMS_EXPORT_VALUE(p, path, iters);
                AMGCL_PARAMS_EXPORT_VALUE(p, path, damping);
            }
        } prm;

    public:
        ilu_solve(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D,
                const params &prm = params(),
                const backend_params &bprm = backend_params()
                ) :
            prm(prm),
            L(Backend::copy_matrix(L, bprm)),
            U(Backend::copy_matrix(U, bprm)),
            D(Backend::copy_vector(D, bprm)),
            t1(Backend::create_vector(backend::rows(*L), bprm)),
            t2(Backend::create_vector(backend::rows(*L), bprm))
        {}

        template <class Vector>
        void solve(Vector &x) {
            vector *y0 = t1.get();
            vector *y1 = t2.get();

            backend::axpby(prm.damping, x, 0.0, *y0);
            for(unsigned i = 0; i < prm.iters; ++i) {
                backend::residual(x, *L, *y0, *y1);
                backend::axpby(prm.damping, *y1, (1-prm.damping), *y0);
            }

            backend::vmul(prm.damping, *D, *y0, 0.0, x);
            for(unsigned i = 0; i < prm.iters; ++i) {
                backend::residual(*y0, *U, x, *y1);
                backend::vmul(prm.damping, *D, *y1, (1-prm.damping), x);
            }
        }

    private:
        boost::shared_ptr<matrix> L;
        boost::shared_ptr<matrix> U;
        boost::shared_ptr<matrix_diagonal> D;
        boost::shared_ptr<vector> t1, t2;
};

template <class value_type>
class ilu_solve< backend::builtin<value_type> > {
    public:
        typedef backend::builtin<value_type> Backend;
        typedef typename Backend::params backend_params;
        typedef typename Backend::matrix matrix;
        typedef typename Backend::vector vector;
        typedef typename Backend::matrix_diagonal matrix_diagonal;
        typedef typename backend::builtin<value_type>::matrix build_matrix;
        typedef typename Backend::rhs_type rhs_type;
        typedef typename math::scalar_of<value_type>::type scalar_type;

        struct params {
            /// Use serial version of the algorithm
            bool serial;

            params() : serial(num_threads() < 4) {}

            params(const boost::property_tree::ptree &p)
                : AMGCL_PARAMS_IMPORT_VALUE(p, serial)
            {
                AMGCL_PARAMS_CHECK(p, (serial));
            }

            void get(boost::property_tree::ptree &p, const std::string &path) const {
                AMGCL_PARAMS_EXPORT_VALUE(p, path, serial);
            }
        } prm;

        ilu_solve(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D,
                const params &prm = params(),
                const backend_params& = backend_params()
                ) : prm(prm)
        {
            if (prm.serial)
                serial_init(L, U, D);
            else
                parallel_init(L, U, D);
        }

        template <class Vector>
        void solve(Vector &x) {
            if (prm.serial)
                serial_solve(x);
            else
                parallel_solve(x);
        }

    private:
        bool is_serial;

        static int num_threads() {
#ifdef _OPENMP
            return omp_get_max_threads();
#else
            return 1;
#endif
        }

        static int thread_id() {
#ifdef _OPENMP
            return omp_get_thread_num();
#else
            return 0;
#endif
        }

        // copies of the input matrices for the fallback (serial)
        // implementation:
        boost::shared_ptr<matrix>          L;
        boost::shared_ptr<matrix>          U;
        boost::shared_ptr<matrix_diagonal> D;

        void serial_init(
                boost::shared_ptr<build_matrix>    L,
                boost::shared_ptr<build_matrix>    U,
                boost::shared_ptr<matrix_diagonal> D
                )
        {
            this->L = L;
            this->U = U;
            this->D = D;
        }

        template <class Vector>
        void serial_solve(Vector &x) {
            const size_t n = backend::rows(*L);

            const matrix          &L = *(this->L);
            const matrix          &U = *(this->U);
            const matrix_diagonal &D = *(this->D);

            for(size_t i = 0; i < n; i++) {
                for(ptrdiff_t j = L.ptr[i], e = L.ptr[i+1]; j < e; ++j)
                    x[i] -= L.val[j] * x[L.col[j]];
            }

            for(size_t i = n; i-- > 0;) {
                for(ptrdiff_t j = U.ptr[i], e = U.ptr[i+1]; j < e; ++j)
                    x[i] -= U.val[j] * x[U.col[j]];
                x[i] = D[i] * x[i];
            }
        }

        // OpenMP solver for sparse triangular systems.
        // The solver uses level scheduling approach.
        // Each level (a set of matrix rows that can be computed independently)
        // is split into tasks, a task per thread, and the matrix data is
        // distributed across threads to improve cache and NUMA locality.
        template <bool lower>
        struct sptr_solve {
            // a task is a set of rows that can be computed independently by a
            // single thread.
            struct task {
                ptrdiff_t beg, end; // rows to process

                task(ptrdiff_t beg, ptrdiff_t end) : beg(beg), end(end) {}
            };

            int nthreads;

            // thread-specific storage:
            std::vector< std::vector<task>       > tasks;
            std::vector< std::vector<ptrdiff_t>  > ptr;
            std::vector< std::vector<ptrdiff_t>  > col;
            std::vector< std::vector<value_type> > val;
            std::vector< std::vector<ptrdiff_t>  > ord; // rows ordered by levels
            std::vector< std::vector<value_type> > D;

            template <class Matrix>
            sptr_solve(const Matrix &A, const value_type *_D = 0)
                : nthreads(num_threads()), tasks(nthreads),
                  ptr(nthreads), col(nthreads), val(nthreads), ord(nthreads)
            {
                ptrdiff_t n    = A.nrows;
                ptrdiff_t nlev = 0;

                std::vector<ptrdiff_t> level(n, 0);
                std::vector<ptrdiff_t> order(n, 0);

                // 1. split rows into levels.
                ptrdiff_t beg = lower ? 0 : n-1;
                ptrdiff_t end = lower ? n :  -1;
                ptrdiff_t inc = lower ? 1 :  -1;

                for(ptrdiff_t i = beg; i != end; i += inc) {
                    ptrdiff_t l = level[i];

                    for(ptrdiff_t j = A.ptr[i]; j < A.ptr[i+1]; ++j)
                        l = std::max(l, level[A.col[j]]+1);

                    level[i] = l;
                    nlev = std::max(nlev, l+1);
                }

                // 2. reorder matrix rows.
                std::vector<ptrdiff_t> start(nlev+1, 0);

                for(ptrdiff_t i = 0; i < n; ++i)
                    ++start[level[i]+1];

                std::partial_sum(start.begin(), start.end(), start.begin());

                for(ptrdiff_t i = 0; i < n; ++i)
                    order[start[level[i]]++] = i;

                std::rotate(start.begin(), start.end() - 1, start.end());
                start[0] = 0;

                // 3. Organize matrix rows into tasks.
                //    Each level is split into nthreads tasks.
                std::vector<ptrdiff_t> thread_rows(nthreads, 0);
                std::vector<ptrdiff_t> thread_cols(nthreads, 0);

#pragma omp parallel
                {
                    int tid = thread_id();
                    tasks[tid].reserve(nlev);

                    for(ptrdiff_t lev = 0; lev < nlev; ++lev) {
                        // split each level into tasks.
                        ptrdiff_t lev_size = start[lev+1] - start[lev];
                        ptrdiff_t chunk_size = (lev_size + nthreads - 1) / nthreads;

                        ptrdiff_t beg = std::min(tid * chunk_size, lev_size);
                        ptrdiff_t end = std::min(beg + chunk_size, lev_size);

                        beg += start[lev];
                        end += start[lev];

                        tasks[tid].push_back(task(beg, end));

                        // count rows and nonzeros in the current task
                        thread_rows[tid] += end - beg;
                        for(ptrdiff_t i = beg; i < end; ++i) {
                            ptrdiff_t j = order[i];
                            thread_cols[tid] += A.ptr[j+1] - A.ptr[j];
                        }
                    }
                }

                // 4. reorganize matrix data for better cache and NUMA locality.
                if (!lower) D.resize(nthreads);

#pragma omp parallel
                {
                    int tid = thread_id();

                    col[tid].reserve(thread_cols[tid]);
                    val[tid].reserve(thread_cols[tid]);
                    ord[tid].reserve(thread_rows[tid]);
                    ptr[tid].reserve(thread_rows[tid] + 1);
                    ptr[tid].push_back(0);

                    if (!lower) D[tid].reserve(thread_rows[tid]);

                    BOOST_FOREACH(task &t, tasks[tid]) {
                        ptrdiff_t loc_beg = ptr[tid].size() - 1;
                        ptrdiff_t loc_end = loc_beg;

                        for(ptrdiff_t r = t.beg; r < t.end; ++r, ++loc_end) {
                            ptrdiff_t i = order[r];
                            if (!lower) D[tid].push_back(_D[i]);

                            ord[tid].push_back(i);

                            for(ptrdiff_t j = A.ptr[i]; j < A.ptr[i+1]; ++j) {
                                col[tid].push_back(A.col[j]);
                                val[tid].push_back(A.val[j]);
                            }

                            ptr[tid].push_back(col[tid].size());
                        }

                        t.beg = loc_beg;
                        t.end = loc_end;
                    }
                }
            }

            template <class Vector>
            void solve(Vector &x) const {
#pragma omp parallel
                {
                    int tid = thread_id();

                    BOOST_FOREACH(const task &t, tasks[tid]) {
                        for(ptrdiff_t r = t.beg; r < t.end; ++r) {
                            ptrdiff_t i   = ord[tid][r];
                            ptrdiff_t beg = ptr[tid][r];
                            ptrdiff_t end = ptr[tid][r+1];

                            rhs_type X = math::zero<rhs_type>();
                            for(ptrdiff_t j = beg; j < end; ++j)
                                X += val[tid][j] * x[col[tid][j]];

                            if (lower)
                                x[i] -= X;
                            else
                                x[i] = D[tid][r] * (x[i] - X);
                        }

                        // each task corresponds to a level, so we need
                        // to synchronize across threads at this point:
#pragma omp barrier
                    }
                }
            }
        };

        boost::shared_ptr< sptr_solve<true > > lower;
        boost::shared_ptr< sptr_solve<false> > upper;

        void parallel_init(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D
                )
        {
            lower = boost::make_shared< sptr_solve<true > >(*L, D->data());
            upper = boost::make_shared< sptr_solve<false> >(*U, D->data());
        }

        template <class Vector>
        void parallel_solve(Vector &x) {
            lower->solve(x);
            upper->solve(x);
        }
};

} // namespace detail
} // namespace relaxation
} // namespace amgcl

#endif
<commit_msg>Drop unused member variable<commit_after>#ifndef AMGCL_RELAXATION_DETAIL_ILU_SOLVE_HPP
#define AMGCL_RELAXATION_DETAIL_ILU_SOLVE_HPP

/*
The MIT License

Copyright (c) 2012-2017 Denis Demidov <dennis.demidov@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

/**
 * \file   amgcl/relaxation/detail/ilu_solve.hpp
 * \author Denis Demidov <dennis.demidov@gmail.com>
 * \brief  Solver for sparse triangular systems obtained as a result of an
 *         incomplete LU factorization.
 */

#include <amgcl/backend/interface.hpp>
#include <amgcl/util.hpp>

namespace amgcl {
namespace relaxation {
namespace detail {

template <class Backend>
class ilu_solve {
    public:
        typedef typename Backend::params backend_params;
        typedef typename Backend::value_type value_type;
        typedef typename Backend::matrix matrix;
        typedef typename Backend::vector vector;
        typedef typename Backend::matrix_diagonal matrix_diagonal;
        typedef typename backend::builtin<value_type>::matrix build_matrix;
        typedef typename math::scalar_of<value_type>::type scalar_type;

        struct params {
            /// Number of Jacobi iterations.
            unsigned    iters;

            /// Damping factor.
            scalar_type damping;

            params() : iters(2), damping(0.72) {}

            params(const boost::property_tree::ptree &p)
                : AMGCL_PARAMS_IMPORT_VALUE(p, iters)
                , AMGCL_PARAMS_IMPORT_VALUE(p, damping)
            {
                AMGCL_PARAMS_CHECK(p, (iters)(damping));
            }

            void get(boost::property_tree::ptree &p, const std::string &path) const {
                AMGCL_PARAMS_EXPORT_VALUE(p, path, iters);
                AMGCL_PARAMS_EXPORT_VALUE(p, path, damping);
            }
        } prm;

    public:
        ilu_solve(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D,
                const params &prm = params(),
                const backend_params &bprm = backend_params()
                ) :
            prm(prm),
            L(Backend::copy_matrix(L, bprm)),
            U(Backend::copy_matrix(U, bprm)),
            D(Backend::copy_vector(D, bprm)),
            t1(Backend::create_vector(backend::rows(*L), bprm)),
            t2(Backend::create_vector(backend::rows(*L), bprm))
        {}

        template <class Vector>
        void solve(Vector &x) {
            vector *y0 = t1.get();
            vector *y1 = t2.get();

            backend::axpby(prm.damping, x, 0.0, *y0);
            for(unsigned i = 0; i < prm.iters; ++i) {
                backend::residual(x, *L, *y0, *y1);
                backend::axpby(prm.damping, *y1, (1-prm.damping), *y0);
            }

            backend::vmul(prm.damping, *D, *y0, 0.0, x);
            for(unsigned i = 0; i < prm.iters; ++i) {
                backend::residual(*y0, *U, x, *y1);
                backend::vmul(prm.damping, *D, *y1, (1-prm.damping), x);
            }
        }

    private:
        boost::shared_ptr<matrix> L;
        boost::shared_ptr<matrix> U;
        boost::shared_ptr<matrix_diagonal> D;
        boost::shared_ptr<vector> t1, t2;
};

template <class value_type>
class ilu_solve< backend::builtin<value_type> > {
    public:
        typedef backend::builtin<value_type> Backend;
        typedef typename Backend::params backend_params;
        typedef typename Backend::matrix matrix;
        typedef typename Backend::vector vector;
        typedef typename Backend::matrix_diagonal matrix_diagonal;
        typedef typename backend::builtin<value_type>::matrix build_matrix;
        typedef typename Backend::rhs_type rhs_type;
        typedef typename math::scalar_of<value_type>::type scalar_type;

        struct params {
            /// Use serial version of the algorithm
            bool serial;

            params() : serial(num_threads() < 4) {}

            params(const boost::property_tree::ptree &p)
                : AMGCL_PARAMS_IMPORT_VALUE(p, serial)
            {
                AMGCL_PARAMS_CHECK(p, (serial));
            }

            void get(boost::property_tree::ptree &p, const std::string &path) const {
                AMGCL_PARAMS_EXPORT_VALUE(p, path, serial);
            }
        } prm;

        ilu_solve(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D,
                const params &prm = params(),
                const backend_params& = backend_params()
                ) : prm(prm)
        {
            if (prm.serial)
                serial_init(L, U, D);
            else
                parallel_init(L, U, D);
        }

        template <class Vector>
        void solve(Vector &x) {
            if (prm.serial)
                serial_solve(x);
            else
                parallel_solve(x);
        }

    private:
        static int num_threads() {
#ifdef _OPENMP
            return omp_get_max_threads();
#else
            return 1;
#endif
        }

        static int thread_id() {
#ifdef _OPENMP
            return omp_get_thread_num();
#else
            return 0;
#endif
        }

        // copies of the input matrices for the fallback (serial)
        // implementation:
        boost::shared_ptr<matrix>          L;
        boost::shared_ptr<matrix>          U;
        boost::shared_ptr<matrix_diagonal> D;

        void serial_init(
                boost::shared_ptr<build_matrix>    L,
                boost::shared_ptr<build_matrix>    U,
                boost::shared_ptr<matrix_diagonal> D
                )
        {
            this->L = L;
            this->U = U;
            this->D = D;
        }

        template <class Vector>
        void serial_solve(Vector &x) {
            const size_t n = backend::rows(*L);

            const matrix          &L = *(this->L);
            const matrix          &U = *(this->U);
            const matrix_diagonal &D = *(this->D);

            for(size_t i = 0; i < n; i++) {
                for(ptrdiff_t j = L.ptr[i], e = L.ptr[i+1]; j < e; ++j)
                    x[i] -= L.val[j] * x[L.col[j]];
            }

            for(size_t i = n; i-- > 0;) {
                for(ptrdiff_t j = U.ptr[i], e = U.ptr[i+1]; j < e; ++j)
                    x[i] -= U.val[j] * x[U.col[j]];
                x[i] = D[i] * x[i];
            }
        }

        // OpenMP solver for sparse triangular systems.
        // The solver uses level scheduling approach.
        // Each level (a set of matrix rows that can be computed independently)
        // is split into tasks, a task per thread, and the matrix data is
        // distributed across threads to improve cache and NUMA locality.
        template <bool lower>
        struct sptr_solve {
            // a task is a set of rows that can be computed independently by a
            // single thread.
            struct task {
                ptrdiff_t beg, end; // rows to process

                task(ptrdiff_t beg, ptrdiff_t end) : beg(beg), end(end) {}
            };

            int nthreads;

            // thread-specific storage:
            std::vector< std::vector<task>       > tasks;
            std::vector< std::vector<ptrdiff_t>  > ptr;
            std::vector< std::vector<ptrdiff_t>  > col;
            std::vector< std::vector<value_type> > val;
            std::vector< std::vector<ptrdiff_t>  > ord; // rows ordered by levels
            std::vector< std::vector<value_type> > D;

            template <class Matrix>
            sptr_solve(const Matrix &A, const value_type *_D = 0)
                : nthreads(num_threads()), tasks(nthreads),
                  ptr(nthreads), col(nthreads), val(nthreads), ord(nthreads)
            {
                ptrdiff_t n    = A.nrows;
                ptrdiff_t nlev = 0;

                std::vector<ptrdiff_t> level(n, 0);
                std::vector<ptrdiff_t> order(n, 0);

                // 1. split rows into levels.
                ptrdiff_t beg = lower ? 0 : n-1;
                ptrdiff_t end = lower ? n :  -1;
                ptrdiff_t inc = lower ? 1 :  -1;

                for(ptrdiff_t i = beg; i != end; i += inc) {
                    ptrdiff_t l = level[i];

                    for(ptrdiff_t j = A.ptr[i]; j < A.ptr[i+1]; ++j)
                        l = std::max(l, level[A.col[j]]+1);

                    level[i] = l;
                    nlev = std::max(nlev, l+1);
                }

                // 2. reorder matrix rows.
                std::vector<ptrdiff_t> start(nlev+1, 0);

                for(ptrdiff_t i = 0; i < n; ++i)
                    ++start[level[i]+1];

                std::partial_sum(start.begin(), start.end(), start.begin());

                for(ptrdiff_t i = 0; i < n; ++i)
                    order[start[level[i]]++] = i;

                std::rotate(start.begin(), start.end() - 1, start.end());
                start[0] = 0;

                // 3. Organize matrix rows into tasks.
                //    Each level is split into nthreads tasks.
                std::vector<ptrdiff_t> thread_rows(nthreads, 0);
                std::vector<ptrdiff_t> thread_cols(nthreads, 0);

#pragma omp parallel
                {
                    int tid = thread_id();
                    tasks[tid].reserve(nlev);

                    for(ptrdiff_t lev = 0; lev < nlev; ++lev) {
                        // split each level into tasks.
                        ptrdiff_t lev_size = start[lev+1] - start[lev];
                        ptrdiff_t chunk_size = (lev_size + nthreads - 1) / nthreads;

                        ptrdiff_t beg = std::min(tid * chunk_size, lev_size);
                        ptrdiff_t end = std::min(beg + chunk_size, lev_size);

                        beg += start[lev];
                        end += start[lev];

                        tasks[tid].push_back(task(beg, end));

                        // count rows and nonzeros in the current task
                        thread_rows[tid] += end - beg;
                        for(ptrdiff_t i = beg; i < end; ++i) {
                            ptrdiff_t j = order[i];
                            thread_cols[tid] += A.ptr[j+1] - A.ptr[j];
                        }
                    }
                }

                // 4. reorganize matrix data for better cache and NUMA locality.
                if (!lower) D.resize(nthreads);

#pragma omp parallel
                {
                    int tid = thread_id();

                    col[tid].reserve(thread_cols[tid]);
                    val[tid].reserve(thread_cols[tid]);
                    ord[tid].reserve(thread_rows[tid]);
                    ptr[tid].reserve(thread_rows[tid] + 1);
                    ptr[tid].push_back(0);

                    if (!lower) D[tid].reserve(thread_rows[tid]);

                    BOOST_FOREACH(task &t, tasks[tid]) {
                        ptrdiff_t loc_beg = ptr[tid].size() - 1;
                        ptrdiff_t loc_end = loc_beg;

                        for(ptrdiff_t r = t.beg; r < t.end; ++r, ++loc_end) {
                            ptrdiff_t i = order[r];
                            if (!lower) D[tid].push_back(_D[i]);

                            ord[tid].push_back(i);

                            for(ptrdiff_t j = A.ptr[i]; j < A.ptr[i+1]; ++j) {
                                col[tid].push_back(A.col[j]);
                                val[tid].push_back(A.val[j]);
                            }

                            ptr[tid].push_back(col[tid].size());
                        }

                        t.beg = loc_beg;
                        t.end = loc_end;
                    }
                }
            }

            template <class Vector>
            void solve(Vector &x) const {
#pragma omp parallel
                {
                    int tid = thread_id();

                    BOOST_FOREACH(const task &t, tasks[tid]) {
                        for(ptrdiff_t r = t.beg; r < t.end; ++r) {
                            ptrdiff_t i   = ord[tid][r];
                            ptrdiff_t beg = ptr[tid][r];
                            ptrdiff_t end = ptr[tid][r+1];

                            rhs_type X = math::zero<rhs_type>();
                            for(ptrdiff_t j = beg; j < end; ++j)
                                X += val[tid][j] * x[col[tid][j]];

                            if (lower)
                                x[i] -= X;
                            else
                                x[i] = D[tid][r] * (x[i] - X);
                        }

                        // each task corresponds to a level, so we need
                        // to synchronize across threads at this point:
#pragma omp barrier
                    }
                }
            }
        };

        boost::shared_ptr< sptr_solve<true > > lower;
        boost::shared_ptr< sptr_solve<false> > upper;

        void parallel_init(
                boost::shared_ptr<build_matrix> L,
                boost::shared_ptr<build_matrix> U,
                boost::shared_ptr<backend::numa_vector<value_type> > D
                )
        {
            lower = boost::make_shared< sptr_solve<true > >(*L, D->data());
            upper = boost::make_shared< sptr_solve<false> >(*U, D->data());
        }

        template <class Vector>
        void parallel_solve(Vector &x) {
            lower->solve(x);
            upper->solve(x);
        }
};

} // namespace detail
} // namespace relaxation
} // namespace amgcl

#endif
<|endoftext|>


<commit_before>#include "fastqreader.hpp"

#include <random>
#include <fstream>
#include <iostream>
#include <iostream>

#include "sequence.hpp"

namespace OLC
{

FASTQReader::FASTQReader(const std::string& filename)
  : Reader(),
    filename_(filename)
{

}

FASTQReader::~FASTQReader() {

}

const std::vector<std::unique_ptr<OLC::Sequence>> FASTQReader::readSequences()
{
  std::ifstream input(filename_);
  std::vector<std::unique_ptr<Sequence>> sequences;

  // TODO (josko): throw?

  if (input.fail())
    return sequences;

  while (!input.eof())
  {
    std::string identifier;
    std::getline(input, identifier);

    // invalid line, try to start over

    if (identifier.empty() || identifier.at(0) != '@')
      continue;

    // cut the initial '@' off

    identifier = identifier.substr(1);

    std::string description;
    const std::size_t delimiterIndex = identifier.find('|');

    if (delimiterIndex != std::string::npos)
    {
      description = identifier.substr(delimiterIndex + 1);
      identifier = identifier.substr(0, delimiterIndex);
    }

    std::string sequence;
    std::string nucleotides;
    std::random_device rd;
    std::mt19937 generator(rd());
    std::uniform_int_distribution<> distribution(0, 3);
    std::unique_ptr<Nucleotides> nucleotideSequence(new Nucleotides());
    nucleotideSequence.get()->reserve(sequence.size());
    std::getline(input, sequence);

    for (const auto c : sequence)
    {
      switch (c)
      {
        case 'A': nucleotideSequence.get()->push_back(NucleotideLetter::A); break;
        case 'T': nucleotideSequence.get()->push_back(NucleotideLetter::T); break;
        case 'C': nucleotideSequence.get()->push_back(NucleotideLetter::C); break;
        case 'G': nucleotideSequence.get()->push_back(NucleotideLetter::G); break;
        case '-':
        {
          const uint32_t randomNucleotide = distribution(generator);
          const NucleotideLetter nucleotide = static_cast<NucleotideLetter>(randomNucleotide);
          nucleotideSequence.get()->push_back(nucleotide);
          break;
        }
        default:
        {
          std::cout << "ERROR: Unrecognized nucleotide " << c << " detected\n";
          break;
        }
      }
    }

    std::string plus;
    std::getline(input, plus);

    // invalid line, try to start over

    if (plus.empty() || plus.at(0) != '+')
      continue;

    // quality is ignored

    std::string quality;
    std::getline(input, quality);

    if (!identifier.empty() && !sequence.empty() && !plus.empty() && !quality.empty())
    {
      std::unique_ptr<Sequence> pointer(new Sequence(identifier, description, std::move(nucleotideSequence)));
      sequences.push_back(std::move(pointer));
    }
  }

  input.close();

  return sequences;
}

}

<commit_msg>fastqreader: don't read the quality string but rather ignore it<commit_after>#include "fastqreader.hpp"

#include <random>
#include <fstream>
#include <iostream>
#include <iostream>

#include "sequence.hpp"

namespace OLC
{

FASTQReader::FASTQReader(const std::string& filename)
  : Reader(),
    filename_(filename)
{

}

FASTQReader::~FASTQReader() {

}

const std::vector<std::unique_ptr<OLC::Sequence>> FASTQReader::readSequences()
{
  std::ifstream input(filename_);
  std::vector<std::unique_ptr<Sequence>> sequences;

  // TODO (josko): throw?

  if (input.fail())
    return sequences;

  while (!input.eof())
  {
    std::string identifier;
    std::getline(input, identifier);

    // invalid line, try to start over

    if (identifier.empty() || identifier.at(0) != '@')
      continue;

    // cut the initial '@' off

    identifier = identifier.substr(1);

    std::string description;
    const std::size_t delimiterIndex = identifier.find('|');

    if (delimiterIndex != std::string::npos)
    {
      description = identifier.substr(delimiterIndex + 1);
      identifier = identifier.substr(0, delimiterIndex);
    }

    std::string sequence;
    std::string nucleotides;
    std::random_device rd;
    std::mt19937 generator(rd());
    std::uniform_int_distribution<> distribution(0, 3);
    std::unique_ptr<Nucleotides> nucleotideSequence(new Nucleotides());
    nucleotideSequence.get()->reserve(sequence.size());
    std::getline(input, sequence);

    for (const auto c : sequence)
    {
      switch (c)
      {
        case 'A': nucleotideSequence.get()->push_back(NucleotideLetter::A); break;
        case 'T': nucleotideSequence.get()->push_back(NucleotideLetter::T); break;
        case 'C': nucleotideSequence.get()->push_back(NucleotideLetter::C); break;
        case 'G': nucleotideSequence.get()->push_back(NucleotideLetter::G); break;
        case '-':
        {
          const uint32_t randomNucleotide = distribution(generator);
          const NucleotideLetter nucleotide = static_cast<NucleotideLetter>(randomNucleotide);
          nucleotideSequence.get()->push_back(nucleotide);
          break;
        }
        default:
        {
          std::cout << "ERROR: Unrecognized nucleotide " << c << " detected\n";
          break;
        }
      }
    }

    std::string plus;
    std::getline(input, plus);

    // invalid line, try to start over

    if (plus.empty() || plus.at(0) != '+')
      continue;

    // ignore quality line

    input.ignore(std::numeric_limits<std::streamsize>::max(), '\n');

    if (!identifier.empty() && !sequence.empty() && !plus.empty())
    {
      std::unique_ptr<Sequence> pointer(new Sequence(identifier, description, std::move(nucleotideSequence)));
      sequences.push_back(std::move(pointer));
    }
  }

  input.close();

  return sequences;
}

}

<|endoftext|>


<commit_before>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#include <boost/test/unit_test.hpp>
#include "Grappa.hpp"
#include "ParallelLoop.hpp"
#include "GlobalAllocator.hpp"
#include "Delegate.hpp"
#include "GlobalVector.hpp"
#include "Statistics.hpp"

#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/variate_generator.hpp>

using namespace Grappa;

BOOST_AUTO_TEST_SUITE( GlobalVector_tests );

static size_t N = (1L<<10) - 21;

DEFINE_int64(nelems, N, "number of elements in (large) test arrays");
DEFINE_int64(buffer_size, 1<<10, "number of elements in buffer");

DEFINE_bool(perf, false, "do performance test");

template< typename T >
inline T next_random() {
  using engine_t = boost::mt19937_64;
  using dist_t = boost::uniform_int<T>;
  using gen_t = boost::variate_generator<engine_t&,dist_t>;

  // continue using same generator with multiple calls (to not repeat numbers)
  // but start at different seed on each node so we don't get overlap
  static engine_t engine(12345L*mycore());
  static gen_t gen(engine, dist_t(0, std::numeric_limits<T>::max()));
  return gen();
}

template< bool             RANDOM = true,
          CompletionEvent* CE     = &impl::local_ce,
          int64_t          TH     = impl::USE_LOOP_THRESHOLD_FLAG >
double push_perf_test() {
  auto qa = GlobalVector<int64_t>::create(N, FLAGS_buffer_size);

  double t = Grappa_walltime();

  forall_global_private<CE,TH>(0, N, [qa](int64_t s, int64_t n){
    for (int64_t i=s; i<s+n; i++) {
      if (RANDOM) {
        qa->push(next_random<int64_t>());
      } else {
        qa->push(42);
      }
    }
  });

  t = Grappa_walltime() - t;

  BOOST_CHECK_EQUAL(qa->size(), N);
  qa->destroy();
  return t;
}

void test_global_vector() {
  BOOST_MESSAGE("Testing GlobalVector"); VLOG(1) << "testing global queue";

  auto qa = GlobalVector<int64_t>::create(N, FLAGS_buffer_size);
  VLOG(1) << "queue addr: " << qa;

  BOOST_CHECK_EQUAL(qa->empty(), true);

  on_all_cores([qa] {
    auto f = [qa](int64_t s, int64_t n) {
      for (int64_t i=s; i<s+n; i++) {
        qa->push(7);
      }
      BOOST_CHECK_EQUAL(qa->empty(), false);
    };
    switch (mycore()) {
    case 0:
      forall_here(0, N/2, f);
      break;
    case 1:
      forall_here(N/2, N-N/2, f);
      break;
    }
  });

  for (int i=0; i<N; i++) {
    BOOST_CHECK_EQUAL(delegate::read(qa->storage()+i), 7);
  }

  forall_localized(qa->begin(), qa->size(), [](int64_t i, int64_t& e) { e = 9; });

  for (int i=0; i<N; i++) {
    BOOST_CHECK_EQUAL(delegate::read(qa->storage()+i), 9);
  }

  qa->destroy();
}

void user_main( void * ignore ) {
  if (FLAGS_perf) {
    double t;
    t = push_perf_test<true>();
    LOG(INFO) << "push_time: " << t;

    t = push_perf_test<false>();
    LOG(INFO) << "push_uniform_time: " << t;

  } else {
    test_global_vector();
  }

  Statistics::merge_and_print();
}

BOOST_AUTO_TEST_CASE( test1 ) {

  Grappa_init( &(boost::unit_test::framework::master_test_suite().argc),
                &(boost::unit_test::framework::master_test_suite().argv) );

  Grappa_activate();
  N = FLAGS_nelems;

  Grappa_run_user_main( &user_main, (void*)NULL );

  Grappa_finish( 0 );
}

BOOST_AUTO_TEST_SUITE_END();

<commit_msg>Fix perf part of GlobalVector_tests and add `--ntrials`<commit_after>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#include <boost/test/unit_test.hpp>
#include "Grappa.hpp"
#include "ParallelLoop.hpp"
#include "GlobalAllocator.hpp"
#include "Delegate.hpp"
#include "GlobalVector.hpp"
#include "Statistics.hpp"

#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/variate_generator.hpp>

using namespace Grappa;

BOOST_AUTO_TEST_SUITE( GlobalVector_tests );

static size_t N = (1L<<10) - 21;

DEFINE_int64(nelems, N, "number of elements in (large) test arrays");
DEFINE_int64(buffer_size, 1<<10, "number of elements in buffer");
DEFINE_int64(ntrials, 1, "number of independent trials to average over");

DEFINE_bool(perf, false, "do performance test");

GRAPPA_DEFINE_STAT(SummarizingStatistic<double>, push_time, 0);
GRAPPA_DEFINE_STAT(SummarizingStatistic<double>, push_const_time, 0);

template< typename T >
inline T next_random() {
  using engine_t = boost::mt19937_64;
  using dist_t = boost::uniform_int<T>;
  using gen_t = boost::variate_generator<engine_t&,dist_t>;

  // continue using same generator with multiple calls (to not repeat numbers)
  // but start at different seed on each node so we don't get overlap
  static engine_t engine(12345L*mycore());
  static gen_t gen(engine, dist_t(0, std::numeric_limits<T>::max()));
  return gen();
}

template< bool             RANDOM = true,
          CompletionEvent* CE     = &impl::local_ce,
          int64_t          TH     = impl::USE_LOOP_THRESHOLD_FLAG >
double push_perf_test(GlobalAddress<GlobalVector<int64_t>> qa) {
  qa->clear();
  double t = Grappa_walltime();

  forall_global_private<CE,TH>(0, N, [qa](int64_t s, int64_t n){
    for (int64_t i=s; i<s+n; i++) {
      if (RANDOM) {
        qa->push(next_random<int64_t>());
      } else {
        qa->push(42);
      }
    }
  });

  t = Grappa_walltime() - t;

  BOOST_CHECK_EQUAL(qa->size(), N);
  return t;
}

void test_global_vector() {
  BOOST_MESSAGE("Testing GlobalVector"); VLOG(1) << "testing global queue";

  auto qa = GlobalVector<int64_t>::create(N, FLAGS_buffer_size);
  VLOG(1) << "queue addr: " << qa;

  BOOST_CHECK_EQUAL(qa->empty(), true);

  on_all_cores([qa] {
    auto f = [qa](int64_t s, int64_t n) {
      for (int64_t i=s; i<s+n; i++) {
        qa->push(7);
      }
      BOOST_CHECK_EQUAL(qa->empty(), false);
    };
    switch (mycore()) {
    case 0:
      forall_here(0, N/2, f);
      break;
    case 1:
      forall_here(N/2, N-N/2, f);
      break;
    }
  });

  for (int i=0; i<N; i++) {
    BOOST_CHECK_EQUAL(delegate::read(qa->storage()+i), 7);
  }

  forall_localized(qa->begin(), qa->size(), [](int64_t i, int64_t& e) { e = 9; });

  for (int i=0; i<N; i++) {
    BOOST_CHECK_EQUAL(delegate::read(qa->storage()+i), 9);
  }

  qa->destroy();
}

void user_main( void * ignore ) {
  if (FLAGS_perf) {
    double t;
    auto qa = GlobalVector<int64_t>::create(N, FLAGS_buffer_size);

    for (int i=0; i<FLAGS_ntrials; i++) {
      t = push_perf_test<true>(qa);
      LOG_FIRST_N(INFO,1) << "push_time: " << t;
      push_time += t;

      t = push_perf_test<false>(qa);
      LOG_FIRST_N(INFO,1) << "push_const_time: " << t;
      push_const_time += t;
    }

    qa->destroy();

  } else {
    test_global_vector();
  }

  Statistics::merge_and_print();
}

BOOST_AUTO_TEST_CASE( test1 ) {

  Grappa_init( &(boost::unit_test::framework::master_test_suite().argc),
                &(boost::unit_test::framework::master_test_suite().argv) );

  Grappa_activate();
  N = FLAGS_nelems;

  Grappa_run_user_main( &user_main, (void*)NULL );

  Grappa_finish( 0 );
}

BOOST_AUTO_TEST_SUITE_END();

<|endoftext|>


<commit_before>/** \file   PdfUtil.cc
 *  \brief  Implementation of functions relating to PDF documents.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015-2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "PdfUtil.h"
#include <leptonica/allheaders.h>
#include <tesseract/baseapi.h>
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "util.h"

namespace PdfUtil {

bool ExtractText(const std::string &pdf_document, std::string * const extracted_text,
                 const std::string &start_page, const std::string &end_page)
{
    static std::string pdftotext_path;
    if (pdftotext_path.empty())
        pdftotext_path = ExecUtil::LocateOrDie("pdftotext");

    const FileUtil::AutoTempFile auto_temp_file1;
    const std::string &input_filename(auto_temp_file1.getFilePath());
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("can't write document to \"" + input_filename + "\"!");
        return false;
    }

    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &output_filename(auto_temp_file2.getFilePath());
    std::vector<std::string> pdftotext_params { "-enc", "UTF-8", "-nopgbrk" };
    if (not start_page.empty())
        pdftotext_params.insert(pdftotext_params.end(), { "-f", start_page });
    if (not end_page.empty())
        pdftotext_params.insert(pdftotext_params.end(), { "-l", end_page });
    pdftotext_params.insert(pdftotext_params.end(), { input_filename, output_filename });
    const int retval(ExecUtil::Exec(pdftotext_path, pdftotext_params));
    if (retval != 0) {
        LOG_WARNING("failed to execute \"" + pdftotext_path + "\"!");
        return false;
    }

    if (not FileUtil::ReadString(output_filename, extracted_text)) {
        LOG_WARNING("failed to read extracted text from \"" + output_filename + "\"!");
        return false;
    }

    return not extracted_text->empty();
}

bool PdfFileContainsNoText(const std::string &path) {
    static std::string pdffonts_path;
    if (pdffonts_path.empty())
        pdffonts_path = ExecUtil::LocateOrDie("pdffonts");

    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &output_filename(auto_temp_file.getFilePath());
    const int retval(ExecUtil::Exec(pdffonts_path, { path }, "", output_filename));
    if (retval == 0) {
        std::string output;
        if (not FileUtil::ReadString(output_filename, &output))
            return false;
        return output.length() == 188; // Header only?
    }

    return retval == 0;
}

bool PdfDocContainsNoText(const std::string &document) {
    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &output_filename(auto_temp_file.getFilePath());
    if (not FileUtil::WriteString(output_filename, document))
        return false;
    return PdfFileContainsNoText(output_filename);
}

bool GetTextFromImage(const std::string &img_path, const std::string &tesseract_language_code,
                      std::string * const extracted_text)
{
    static std::string tesseract_path(ExecUtil::LocateOrDie("tesseract"));
    extracted_text->clear();
    std::string stderr_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdoutAndStderr(
            tesseract_path, { img_path, "stdout" /*tesseract arg to redirect*/,
            "-l", tesseract_language_code,
            "--oem", "0" /*use legacy extract to address problems with 4.0 default engine*/},
            extracted_text, &stderr_output, 0 /* timeout */, SIGKILL,
            { {"OMP_THREAD_LIMIT", "1" } }, /* address tesseract 4 IPC problems */
            "" /* working dir */)
        )
        LOG_WARNING("While processing " + img_path + ": " + stderr_output);

    return not extracted_text->empty();
}

bool GetTextFromImagePDF(const std::string &pdf_document, const std::string &tesseract_language_code,
                         std::string * const extracted_text, const unsigned timeout)
{
    extracted_text->clear();

    static std::string pdf_images_script_path(ExecUtil::LocateOrDie("pdfimages"));

    const FileUtil::AutoTempDirectory auto_temp_dir;
    const std::string &output_dirname(auto_temp_dir.getDirectoryPath());
    const std::string input_filename(output_dirname + "/in.pdf");
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("failed to write the PDF to a temp file!");
        return false;
    }

    if (ExecUtil::Exec(pdf_images_script_path, { input_filename, output_dirname + "/out" }, "", "", "", timeout) != 0) {
        LOG_WARNING("failed to extract images from PDF file!");
        return false;
    }

    std::vector<std::string> pdf_image_filenames;
    if (FileUtil::GetFileNameList("out.*", &pdf_image_filenames, output_dirname) == 0) {
        LOG_WARNING("PDF did not contain any images!");
        return false;
    }

    for (const std::string &pdf_image_filename : pdf_image_filenames) {
        std::string image_text;
        if (not GetTextFromImage(output_dirname + "/" + pdf_image_filename, tesseract_language_code, &image_text)) {
            LOG_WARNING("failed to extract text from image " + pdf_image_filename);
            return false;
        }
        *extracted_text += " " + image_text;
    }

    *extracted_text = StringUtil::TrimWhite(*extracted_text);
    return not extracted_text->empty();
}

bool GetOCRedTextFromPDF(const std::string &pdf_document_path, const std::string &tesseract_language_code,
                         std::string * const extracted_text, const unsigned timeout)
{
    extracted_text->clear();
    static std::string pdf_to_image_command(ExecUtil::LocateOrDie("convert"));
    const FileUtil::AutoTempDirectory auto_temp_dir;
    const std::string &image_dirname(auto_temp_dir.getDirectoryPath());
    const std::string temp_image_location = image_dirname + "/img.tiff";
    if (ExecUtil::Exec(pdf_to_image_command, { "-density", "300", pdf_document_path, "-depth", "8", "-strip",
                                               "-background", "white", "-alpha", "off", temp_image_location
                                             }, "", "", "", timeout) != 0) {
        LOG_WARNING("failed to convert PDF to image!");
        return false;
    }
    if (not GetTextFromImage(temp_image_location, tesseract_language_code, extracted_text))
        LOG_WARNING("failed to extract OCRed text");

    *extracted_text = StringUtil::TrimWhite(*extracted_text);
    return not extracted_text->empty();
}

bool ExtractPDFInfo(const std::string &pdf_document, std::string * const pdf_output) {
    static std::string pdfinfo_path;
    if (pdfinfo_path.empty())
        pdfinfo_path = ExecUtil::LocateOrDie("pdfinfo");
    const FileUtil::AutoTempFile auto_temp_file1;
    const std::string &input_filename(auto_temp_file1.getFilePath());
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("can't write document to \"" + input_filename + "\"!");
        return false;
    }
    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &pdfinfo_output_filename(auto_temp_file2.getFilePath());

    const std::vector<std::string> pdfinfo_params { input_filename };
    const int retval(ExecUtil::Exec(pdfinfo_path, pdfinfo_params, pdfinfo_output_filename /* stdout */,
                     pdfinfo_output_filename /* stderr */));
    if (retval != 0) {
        LOG_WARNING("failed to execute \"" + pdfinfo_path + "\"!");
        return false;
    }
    std::string pdfinfo_output;
    if (unlikely(not FileUtil::ReadString(pdfinfo_output_filename, pdf_output)))
        LOG_ERROR("Unable to extract pdfinfo output");

    return true;
}

bool ExtractHTMLAsPages(const std::string &pdf_document, const std::string &output_dirname) {
    static std::string pdftohtml_path;
    if (pdftohtml_path.empty())
        pdftohtml_path = ExecUtil::LocateOrDie("pdftohtml");

    std::vector<std::string> pdftohtml_params { "-i" /* ignore images */,
                                                "-c" /* generate complex output */,
                                                "-hidden" /* force hidden text extraction */,
                                                "-fontfullname" /* outputs the font name without any substitutions */
                                              };
    const std::string pdf_temp_link(output_dirname + '/' + FileUtil::GetBasename(pdf_document));
    FileUtil::CreateSymlink(pdf_document, pdf_temp_link);
    pdftohtml_params.emplace_back(pdf_temp_link);
    ExecUtil::ExecOrDie(pdftohtml_path, pdftohtml_params, "" /* stdin */, "" /* stdout */, "" /* stderr */, 0 /* timeout */,
                        SIGKILL, std::unordered_map<std::string, std::string>() /* env */, output_dirname /* working dir */);

    // Clean up HTML
    static std::string tidy_path;
    if (tidy_path.empty())
        tidy_path = ExecUtil::LocateOrDie("tidy");

    FileUtil::Directory html_pages(output_dirname, ".*\\.html$");
    for (const auto &html_page : html_pages) {
        const std::vector<std::string> tidy_params({ "-modify" /* write back to original file */, "-quiet", html_page.getName() });
        // return code 1 means there were only warnings
        const int tidy_retval(ExecUtil::Exec(tidy_path, tidy_params,  "" /* stdin */, "" /* stdout */, "" /* stderr */, 0 /* timeout */,
                        SIGKILL, std::unordered_map<std::string, std::string>() /* env */, output_dirname /* working dir */));
        if (tidy_retval != 0 and tidy_retval != 1)
            LOG_ERROR("Error while cleaning up " + html_page.getName());

    }

    return true;
}

} // namespace PdfUtil
<commit_msg>Use LSTM<commit_after>/** \file   PdfUtil.cc
 *  \brief  Implementation of functions relating to PDF documents.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015-2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "PdfUtil.h"
#include <leptonica/allheaders.h>
#include <tesseract/baseapi.h>
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "util.h"

namespace PdfUtil {

bool ExtractText(const std::string &pdf_document, std::string * const extracted_text,
                 const std::string &start_page, const std::string &end_page)
{
    static std::string pdftotext_path;
    if (pdftotext_path.empty())
        pdftotext_path = ExecUtil::LocateOrDie("pdftotext");

    const FileUtil::AutoTempFile auto_temp_file1;
    const std::string &input_filename(auto_temp_file1.getFilePath());
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("can't write document to \"" + input_filename + "\"!");
        return false;
    }

    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &output_filename(auto_temp_file2.getFilePath());
    std::vector<std::string> pdftotext_params { "-enc", "UTF-8", "-nopgbrk" };
    if (not start_page.empty())
        pdftotext_params.insert(pdftotext_params.end(), { "-f", start_page });
    if (not end_page.empty())
        pdftotext_params.insert(pdftotext_params.end(), { "-l", end_page });
    pdftotext_params.insert(pdftotext_params.end(), { input_filename, output_filename });
    const int retval(ExecUtil::Exec(pdftotext_path, pdftotext_params));
    if (retval != 0) {
        LOG_WARNING("failed to execute \"" + pdftotext_path + "\"!");
        return false;
    }

    if (not FileUtil::ReadString(output_filename, extracted_text)) {
        LOG_WARNING("failed to read extracted text from \"" + output_filename + "\"!");
        return false;
    }

    return not extracted_text->empty();
}

bool PdfFileContainsNoText(const std::string &path) {
    static std::string pdffonts_path;
    if (pdffonts_path.empty())
        pdffonts_path = ExecUtil::LocateOrDie("pdffonts");

    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &output_filename(auto_temp_file.getFilePath());
    const int retval(ExecUtil::Exec(pdffonts_path, { path }, "", output_filename));
    if (retval == 0) {
        std::string output;
        if (not FileUtil::ReadString(output_filename, &output))
            return false;
        return output.length() == 188; // Header only?
    }

    return retval == 0;
}

bool PdfDocContainsNoText(const std::string &document) {
    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &output_filename(auto_temp_file.getFilePath());
    if (not FileUtil::WriteString(output_filename, document))
        return false;
    return PdfFileContainsNoText(output_filename);
}

bool GetTextFromImage(const std::string &img_path, const std::string &tesseract_language_code,
                      std::string * const extracted_text)
{
    static std::string tesseract_path(ExecUtil::LocateOrDie("tesseract"));
    extracted_text->clear();
    std::string stderr_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdoutAndStderr(
            tesseract_path, { img_path, "stdout" /*tesseract arg to redirect*/,
            "-l", tesseract_language_code,
            "--oem", "1" /*unlike beforehand use LSTM extract instead of legacy */},
            extracted_text, &stderr_output, 0 /* timeout */, SIGKILL,
            { {"OMP_THREAD_LIMIT", "1" } }, /* address tesseract 4 IPC problems */
            "" /* working dir */)
        )
        LOG_WARNING("While processing " + img_path + ": " + stderr_output);

    return not extracted_text->empty();
}

bool GetTextFromImagePDF(const std::string &pdf_document, const std::string &tesseract_language_code,
                         std::string * const extracted_text, const unsigned timeout)
{
    extracted_text->clear();

    static std::string pdf_images_script_path(ExecUtil::LocateOrDie("pdfimages"));

    const FileUtil::AutoTempDirectory auto_temp_dir;
    const std::string &output_dirname(auto_temp_dir.getDirectoryPath());
    const std::string input_filename(output_dirname + "/in.pdf");
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("failed to write the PDF to a temp file!");
        return false;
    }

    if (ExecUtil::Exec(pdf_images_script_path, { input_filename, output_dirname + "/out" }, "", "", "", timeout) != 0) {
        LOG_WARNING("failed to extract images from PDF file!");
        return false;
    }

    std::vector<std::string> pdf_image_filenames;
    if (FileUtil::GetFileNameList("out.*", &pdf_image_filenames, output_dirname) == 0) {
        LOG_WARNING("PDF did not contain any images!");
        return false;
    }

    for (const std::string &pdf_image_filename : pdf_image_filenames) {
        std::string image_text;
        if (not GetTextFromImage(output_dirname + "/" + pdf_image_filename, tesseract_language_code, &image_text)) {
            LOG_WARNING("failed to extract text from image " + pdf_image_filename);
            return false;
        }
        *extracted_text += " " + image_text;
    }

    *extracted_text = StringUtil::TrimWhite(*extracted_text);
    return not extracted_text->empty();
}

bool GetOCRedTextFromPDF(const std::string &pdf_document_path, const std::string &tesseract_language_code,
                         std::string * const extracted_text, const unsigned timeout)
{
    extracted_text->clear();
    static std::string pdf_to_image_command(ExecUtil::LocateOrDie("convert"));
    const FileUtil::AutoTempDirectory auto_temp_dir;
    const std::string &image_dirname(auto_temp_dir.getDirectoryPath());
    const std::string temp_image_location = image_dirname + "/img.tiff";
    if (ExecUtil::Exec(pdf_to_image_command, { "-density", "300", pdf_document_path, "-depth", "8", "-strip",
                                               "-background", "white", "-alpha", "off", temp_image_location
                                             }, "", "", "", timeout) != 0) {
        LOG_WARNING("failed to convert PDF to image!");
        return false;
    }
    if (not GetTextFromImage(temp_image_location, tesseract_language_code, extracted_text))
        LOG_WARNING("failed to extract OCRed text");

    *extracted_text = StringUtil::TrimWhite(*extracted_text);
    return not extracted_text->empty();
}

bool ExtractPDFInfo(const std::string &pdf_document, std::string * const pdf_output) {
    static std::string pdfinfo_path;
    if (pdfinfo_path.empty())
        pdfinfo_path = ExecUtil::LocateOrDie("pdfinfo");
    const FileUtil::AutoTempFile auto_temp_file1;
    const std::string &input_filename(auto_temp_file1.getFilePath());
    if (not FileUtil::WriteString(input_filename, pdf_document)) {
        LOG_WARNING("can't write document to \"" + input_filename + "\"!");
        return false;
    }
    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &pdfinfo_output_filename(auto_temp_file2.getFilePath());

    const std::vector<std::string> pdfinfo_params { input_filename };
    const int retval(ExecUtil::Exec(pdfinfo_path, pdfinfo_params, pdfinfo_output_filename /* stdout */,
                     pdfinfo_output_filename /* stderr */));
    if (retval != 0) {
        LOG_WARNING("failed to execute \"" + pdfinfo_path + "\"!");
        return false;
    }
    std::string pdfinfo_output;
    if (unlikely(not FileUtil::ReadString(pdfinfo_output_filename, pdf_output)))
        LOG_ERROR("Unable to extract pdfinfo output");

    return true;
}

bool ExtractHTMLAsPages(const std::string &pdf_document, const std::string &output_dirname) {
    static std::string pdftohtml_path;
    if (pdftohtml_path.empty())
        pdftohtml_path = ExecUtil::LocateOrDie("pdftohtml");

    std::vector<std::string> pdftohtml_params { "-i" /* ignore images */,
                                                "-c" /* generate complex output */,
                                                "-hidden" /* force hidden text extraction */,
                                                "-fontfullname" /* outputs the font name without any substitutions */
                                              };
    const std::string pdf_temp_link(output_dirname + '/' + FileUtil::GetBasename(pdf_document));
    FileUtil::CreateSymlink(pdf_document, pdf_temp_link);
    pdftohtml_params.emplace_back(pdf_temp_link);
    ExecUtil::ExecOrDie(pdftohtml_path, pdftohtml_params, "" /* stdin */, "" /* stdout */, "" /* stderr */, 0 /* timeout */,
                        SIGKILL, std::unordered_map<std::string, std::string>() /* env */, output_dirname /* working dir */);

    // Clean up HTML
    static std::string tidy_path;
    if (tidy_path.empty())
        tidy_path = ExecUtil::LocateOrDie("tidy");

    FileUtil::Directory html_pages(output_dirname, ".*\\.html$");
    for (const auto &html_page : html_pages) {
        const std::vector<std::string> tidy_params({ "-modify" /* write back to original file */, "-quiet", html_page.getName() });
        // return code 1 means there were only warnings
        const int tidy_retval(ExecUtil::Exec(tidy_path, tidy_params,  "" /* stdin */, "" /* stdout */, "" /* stderr */, 0 /* timeout */,
                        SIGKILL, std::unordered_map<std::string, std::string>() /* env */, output_dirname /* working dir */));
        if (tidy_retval != 0 and tidy_retval != 1)
            LOG_ERROR("Error while cleaning up " + html_page.getName());

    }

    return true;
}

} // namespace PdfUtil
<|endoftext|>


<commit_before>//2014.03.10 - 2014.03.11 Gustaf - CTG.

/* OBJECTIVE :

For our dynamically allocated strings, create a function replaceString that takes
three parameters, each of type arrayString: source, target, and replaceText.

The function replaces every occurrence of target in source with replaceText.

For example,
  if source points to an array containing "abcdabee",
  target points to "ab",
  and replaceText points to "xyz",
  then when the function ends, source should point to an array containing "xyzcdxyzee".

=== PLAN ===

-
-

*/

#include <iostream>

using namespace std;

typedef char *arrayString;

struct posIniNode
{
  int posInitial;

  posIniNode *next; // Pointer to the same struct.
};

typedef posIniNode *posList; // type reserved for the head pointer.

int lengthFunction(arrayString s)
{
  // Determine the length of a string array.
  int count = 0;
  while (s[count] != 0) // not includes the "null terminator".
  {
    count++;
  }
  return count;
}

void identifyLimits (arrayString sourceStr, arrayString targetStr, posList &positionsResult)
{
  // -- SIMULATE result, for simplicity in this test archive.
  //
  // This function works and its code is in the file:
  // cap04-ex4_3-02-test-identify_targets-function.cpp
  // ---

  // New nodes
  posIniNode *newNode1 = new posIniNode;
  newNode1 -> posInitial = 0;
  newNode1 -> next = NULL;

  posIniNode *newNode2 = new posIniNode;
  newNode2 -> posInitial = 4;
  newNode2 -> next = NULL;

  // Linked list
  newNode1 -> next = newNode2;
  positionsResult = newNode1;

  // Cleaning
  newNode1 = NULL;
  newNode2 = NULL;

  // ---

}

void replaceFunction(arrayString &source, arrayString target, arrayString replace)
{

  // -- Get the initial positions of the Target in the Source
  posList resultLimits = NULL; // Head pointer of linked list.
  identifyLimits(source, target, resultLimits);

  // Count
  int countPos = 0;
  posIniNode *countPtr = resultLimits;
  while (countPtr != NULL)
  {
    countPos++; // Number of target string found.
    countPtr = countPtr -> next;
  }

  // --
  int sLength = lengthFunction(source);
  int tLength = lengthFunction(target);
  int rLength = lengthFunction(replace);

  // New size of result string: size of original string,
  // minus the letters for all the targets,
  // plus the new letters, plus the NULL at the end of the string.
  int newSizeSource = sLength - (countPos * tLength) + (countPos * rLength) + 1;
  arrayString newS = new char[newSizeSource];

  posIniNode *loopPtr = resultLimits;
  while (loopPtr != NULL)
  {
    cout << "DEBUG: (Initial position) " << loopPtr -> posInitial << endl;

    source[ loopPtr -> posInitial ] = replace[0];

    loopPtr = loopPtr -> next;
  }
  cout << endl;

  // Free the old value and return the new value
  delete[] source;
  source = newS;

}

void replaceFunctionTester ()
{

  // -- SOURCE STRING
  const int ARRAY_SIZE = 9;
  arrayString s = new char[ARRAY_SIZE];

  s[0] = 'a'; s[1] = 'b'; s[2] = 'c'; s[3] = 'd';
  s[4] = 'a'; s[5] = 'b'; s[6] = 'c'; s[7] = 'e'; s[8] = 0;

  // -- Different tests for the TARGET STRING

  // const int TARGET_SIZE = 9;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 'd';
  // t[4] = 'a'; t[5] = 'b'; t[6] = 'c'; t[7] = 'e'; t[8] = 'f'; t[9] = 0;

  // const int TARGET_SIZE = 8;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 'd';
  // t[4] = 'a'; t[5] = 'b'; t[6] = 'c'; t[7] = 'a'; t[8] = 0;

  // const int TARGET_SIZE = 4;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 0;

  // const int TARGET_SIZE = 3;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'b'; t[1] = 'c'; t[2] = 0;

  const int TARGET_SIZE = 2; arrayString t = new char[TARGET_SIZE];
  t[0] = 'a'; t[1] = 0;

  /// ---

  // -- REPLACE STRING
  // const int REPLACE_SIZE = 4; arrayString r = new char[REPLACE_SIZE];
  // r[0] = 'a'; r[1] = 'b'; r[2] = 'c'; r[3] = 0;

  const int REPLACE_SIZE = 3; arrayString r = new char[REPLACE_SIZE];
  r[0] = 'a'; r[1] = 'b'; r[2] = 0;

  // const int REPLACE_SIZE = 2; arrayString r = new char[REPLACE_SIZE];
  // r[0] = 'x'; r[1] = 0;

  // -- Execution
  cout << "Initial string : " << s << endl;
  cout << "Target string  : " << t << endl;

  replaceFunction(s, t, r);
  cout << "Final string   : " << s << endl;
  cout << endl;

  // Free dynamic memory.
  delete[] s;
  delete[] t;
  delete[] r;

}

int main()
{
  cout << "Variable-Length String Manipulation. Test of REPLACE." << endl;
  replaceFunctionTester();

  cout << endl;
  return 0;
}
<commit_msg>Chapter 04 exercice 4-3 partial progress. Test function that replace a string.<commit_after>//2014.03.15 Gustaf - CTG.

/* OBJECTIVE :

For our dynamically allocated strings, create a function replaceString that takes
three parameters, each of type arrayString: source, target, and replaceText.

The function replaces every occurrence of target in source with replaceText.

For example,
  if source points to an array containing "abcdabee",
  target points to "ab",
  and replaceText points to "xyz",
  then when the function ends, source should point to an array containing "xyzcdxyzee".

=== PLAN ===

ok- Function to replace the string.
  ok- Test 01: target (one letter) and replace (one letter).
  ok- Test 02: target (one letter) and replace (two letter).
  ok- Test 03: target (one letter) and replace (three letter).
  ok- Test 04: target (two letter) and replace (one letter).
  ok- Test 05: target (two letter) and replace (three letter).
  ok- Test 06: target (three letter) and replace (three letter).

*/

#include <iostream>

using namespace std;

typedef char *arrayString;

struct posIniNode
{
  int posInitial;

  posIniNode *next; // Pointer to the same struct.
};

typedef posIniNode *posList; // type reserved for the head pointer.

int lengthFunction(arrayString s)
{
  // Determine the length of a string array.
  int count = 0;
  while (s[count] != 0) // not includes the "null terminator".
  {
    count++;
  }
  return count;
}

void identifyLimits (arrayString sourceStr, arrayString targetStr, posList &positionsResult)
{
  // -- SIMULATE result, for simplicity in this test archive.
  //
  // This function works and its code is in the file:
  // cap04-ex4_3-02-test-identify_targets-function.cpp
  // ---

  // New nodes
  posIniNode *newNode1 = new posIniNode;
  newNode1 -> posInitial = 0;
  newNode1 -> next = NULL;

  posIniNode *newNode2 = new posIniNode;
  newNode2 -> posInitial = 4;
  newNode2 -> next = NULL;

  // Linked list
  newNode1 -> next = newNode2;
  positionsResult = newNode1;

  // Cleaning
  newNode1 = NULL;
  newNode2 = NULL;

}

void replaceFunction(arrayString &source, arrayString target, arrayString replace)
{

  // -- Get the initial positions of the Target in the Source
  posList getResultLimits = NULL; // Head pointer of linked list.
  identifyLimits(source, target, getResultLimits);

  // Count
  int countPos = 0;
  posIniNode *countPtr = getResultLimits;
  while (countPtr != NULL)
  {
    countPos++; // Number of target string found.
    countPtr = countPtr -> next;
  }

  if (countPos == 0)
  {
    cout << "DEBUG: Coincidence not found." << endl;
    return;
  }

  // --
  int sLength = lengthFunction(source);
  int tLength = lengthFunction(target);
  int rLength = lengthFunction(replace);

  // New size for the result string: size of original string,
  //    minus the letters for all the targets,
  //    plus the new letters, plus the NULL at the end of the string.
  int newSizeSource = sLength - (countPos * tLength) + (countPos * rLength) + 1;
  arrayString newS = new char[newSizeSource];
  cout << "DEBUG: (Source length - New Source size): " << sLength << " - " << newSizeSource << endl;

  int i = 0;
  int j = 0;
  int r = 0;
  posIniNode *loopPtr_FOUR = getResultLimits;

  while ( source[i] != 0)
  {
    if ( (loopPtr_FOUR != NULL) && (i == loopPtr_FOUR -> posInitial) )
    {
      cout << "DEBUG: (Initial position) " << loopPtr_FOUR -> posInitial << endl;

      r = 0;
      while (r < rLength)
      {
        cout << "DEBUG: replace " << r << endl;
        newS[j] = replace[r];
        j++;
        r++;
      }

      loopPtr_FOUR = loopPtr_FOUR -> next;
      i = i + tLength; // Jump to the letter after this target.
    }
    else
    {
      newS[j] = source[i];
      i++;
      j++;
    }
  }
  newS[newSizeSource - 1] = 0; // NULL at the end.
  cout << "DEBUG: NULL "  << endl;

  // -- Free the old value and return the new value.
  delete[] source;
  source = newS;

}

void replaceFunctionTester ()
{

  // -- SOURCE STRING
  const int ARRAY_SIZE = 9;
  arrayString s = new char[ARRAY_SIZE];

  s[0] = 'a'; s[1] = 'b'; s[2] = 'c'; s[3] = 'd';
  s[4] = 'a'; s[5] = 'b'; s[6] = 'c'; s[7] = 'e'; s[8] = 0;

  // -- TARGET STRING

  // const int TARGET_SIZE = 9;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 'd';
  // t[4] = 'a'; t[5] = 'b'; t[6] = 'c'; t[7] = 'e'; t[8] = 'f'; t[9] = 0;

  // const int TARGET_SIZE = 8;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 'd';
  // t[4] = 'a'; t[5] = 'b'; t[6] = 'c'; t[7] = 'a'; t[8] = 0;

  // const int TARGET_SIZE = 4;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 'c'; t[3] = 0;

  // const int TARGET_SIZE = 3;
  // arrayString t = new char[TARGET_SIZE];
  // t[0] = 'a'; t[1] = 'b'; t[2] = 0;

  const int TARGET_SIZE = 2; arrayString t = new char[TARGET_SIZE];
  t[0] = 'a'; t[1] = 0;

  // -- REPLACE STRING
  const int REPLACE_SIZE = 4; arrayString r = new char[REPLACE_SIZE];
  r[0] = 'x'; r[1] = 'y'; r[2] = 'z'; r[3] = 0;

  // const int REPLACE_SIZE = 3; arrayString r = new char[REPLACE_SIZE];
  // r[0] = 'x'; r[1] = 'y'; r[2] = 0;

  // const int REPLACE_SIZE = 2; arrayString r = new char[REPLACE_SIZE];
  // r[0] = 'x'; r[1] = 0;

  // -- Execution
  cout << "Initial string : " << s << endl;
  cout << "Target string  : " << t << endl;
  cout << "Replace string : " << r << endl;
  cout << endl;

  replaceFunction(s, t, r);
  cout << "Final string   : " << s << endl;
  cout << endl;

  // Free dynamic memory.
  delete[] s;
  delete[] t;
  delete[] r;

}

int main()
{
  cout << "Variable-Length String Manipulation. Test of REPLACE." << endl;
  replaceFunctionTester();

  cout << endl;
  return 0;
}
<|endoftext|>