<commit_before>/******************************************************************************

  This source file is part of the Avogadro project.

  Copyright 2013 Kitware, Inc.

  This source code is released under the New BSD License, (the "License").

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

******************************************************************************/

#include "gromacsformat.h"

#include <avogadro/core/avogadrocore.h>

#include <avogadro/core/atom.h>
#include <avogadro/core/matrix.h>
#include <avogadro/core/molecule.h>
#include <avogadro/core/unitcell.h>
#include <avogadro/core/utilities.h>

#include <string>
#include <utility>

using Avogadro::Core::Atom;
using Avogadro::Core::Molecule;
using Avogadro::Core::UnitCell;
using Avogadro::Core::lexicalCast;
using Avogadro::Core::trimmed;
using Avogadro::Core::split;

namespace Avogadro {
namespace Io {

GromacsFormat::GromacsFormat()
{
}

GromacsFormat::~GromacsFormat()
{
}

std::vector<std::string> GromacsFormat::fileExtensions() const
{
  return std::vector<std::string>(1, std::string("gro"));
}

std::vector<std::string> GromacsFormat::mimeTypes() const
{
  return std::vector<std::string>(1, std::string("chemical/x-gro"));
}

bool GromacsFormat::read(std::istream &in, Molecule &molecule)
{
  std::string buffer;
  std::string value;

  // Title
  std::getline(in, buffer);
  if (!buffer.empty())
    molecule.setData("name", trimmed(buffer));

  // Atom count
  std::getline(in, buffer);
  buffer = trimmed(buffer);
  bool ok;
  size_t numAtoms = lexicalCast<size_t>(buffer, ok);
  if (buffer.empty() || !ok) {
    appendError("Number of atoms (line 2) invalid.");
    return false;
  }

  // read atom info:
  typedef std::map<std::string, unsigned char> AtomTypeMap;
  AtomTypeMap atomTypes;
  unsigned char customElementCounter = CustomElementMin;
  Vector3 pos;
  while (numAtoms-- > 0) {
    std::getline(in, buffer);
    if (buffer.size() < 44) {
      appendError("Error reading atom specification -- line too short: "
                  + buffer);
      return false;
    }

    // Format of buffer is: (all indices start at 1)
    // Offset:  0 format: %5i   value: Residue number
    // Offset:  5 format: %-5s  value: Residue name
    // Offset: 10 format: %5s   value: Atom name
    // Offset: 15 format: %5i   value: Atom number
    // Offset: 20 format: %8.3f value: x coordinate (nm)
    // Offset: 28 format: %8.3f value: y coordinate (nm)
    // Offset: 36 format: %8.3f value: z coordinate (nm)
    // Offset: 44 format: %8.4f value: x velocity (nm/ps, a.k.a. km/s)
    // Offset: 52 format: %8.4f value: y velocity (nm/ps, a.k.a. km/s)
    // Offset: 60 format: %8.4f value: z velocity (nm/ps, a.k.a. km/s)

    // Atom name:
    value = trimmed(buffer.substr(10, 5));
    AtomTypeMap::const_iterator it = atomTypes.find(value);
    if (it == atomTypes.end()) {
      atomTypes.insert(std::make_pair(value, customElementCounter++));
      it = atomTypes.find(value);
      if (customElementCounter > CustomElementMax) {
        appendError("Custom element type limit exceeded.");
        return false;
      }
    }
    Atom atom = molecule.addAtom(it->second);

    // Coords
    for (int i = 0; i < 3; ++i) {
      value = trimmed(buffer.substr(20 + i * 8, 8));
      pos[i] = lexicalCast<Real>(value, ok);
      if (!ok || value.empty()) {
        appendError("Error reading atom specification -- invalid coordinate: '"
                    + buffer + "' (bad coord: '" + value + "')");
        return false;
      }
    }
    atom.setPosition3d(pos * static_cast<Real>(10.0)); // nm --> Angstrom
  }

  // Set the custom element map if needed:
  if (!atomTypes.empty()) {
    Molecule::CustomElementMap elementMap;
    for (AtomTypeMap::const_iterator it = atomTypes.begin(),
         itEnd = atomTypes.end(); it != itEnd; ++it) {
      elementMap.insert(std::make_pair(it->second, it->first));
    }
    molecule.setCustomElementMap(elementMap);
  }

  // Box description:
  // v1(x) v2(y) v3(z) [v1(y) v1(z) v2(x) v2(z) v3(x) v3(y)]
  // The last six values may be omitted, set all non-specified values to 0.
  // v1(y) == v1(z) == v2(z) == 0 always.
  std::getline(in, buffer);
  std::vector<std::string> tokens(split(buffer, ' ', true));
  if (tokens.size() > 0) {
    if (tokens.size() != 3 && tokens.size() != 9) {
      appendError("Invalid box specification -- need either 3 or 9 values: '"
                  + buffer + "'");
      return false;
    }

    // Index arrays for parsing loop:
    const int rows[] = {0, 1, 2, 1, 2, 0, 2, 0, 1};
    const int cols[] = {0, 1, 2, 0, 0, 1, 1, 2, 2};

    Matrix3 cellMatrix = Matrix3::Zero();
    for (size_t i = 0; i < tokens.size(); ++i) {
      cellMatrix(rows[i], cols[i]) = lexicalCast<Real>(tokens[i], ok);
      if (!ok || tokens[i].empty()) {
        appendError("Invalid box specification -- bad value: '"
                    + tokens[i] + "'");
        return false;
      }
    }

    UnitCell *cell = new UnitCell;
    cell->setCellMatrix(cellMatrix * static_cast<Real>(10)); // nm --> Angstrom
    molecule.setUnitCell(cell);
  }

  return true;
}

bool GromacsFormat::write(std::ostream &, const Core::Molecule &)
{
  return false;
}

} // namespace Io
} // namespace Avogadro
<commit_msg>Added some code to deal with variable decimal places<commit_after>/******************************************************************************

  This source file is part of the Avogadro project.

  Copyright 2013 Kitware, Inc.

  This source code is released under the New BSD License, (the "License").

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

******************************************************************************/

#include "gromacsformat.h"

#include <avogadro/core/avogadrocore.h>

#include <avogadro/core/atom.h>
#include <avogadro/core/matrix.h>
#include <avogadro/core/molecule.h>
#include <avogadro/core/unitcell.h>
#include <avogadro/core/utilities.h>

#include <iostream>

#include <string>
#include <utility>

namespace Avogadro {
namespace Io {

using Core::Atom;
using Core::Molecule;
using Core::UnitCell;
using Core::lexicalCast;
using Core::trimmed;
using Core::split;

using std::string;
using std::getline;
using std::map;
using std::vector;

GromacsFormat::GromacsFormat()
{
}

GromacsFormat::~GromacsFormat()
{
}

std::vector<std::string> GromacsFormat::fileExtensions() const
{
  return std::vector<std::string>(1, std::string("gro"));
}

std::vector<std::string> GromacsFormat::mimeTypes() const
{
  return std::vector<std::string>(1, std::string("chemical/x-gro"));
}

bool GromacsFormat::read(std::istream &in, Molecule &molecule)
{
  string buffer;
  string value;

  // Title
  getline(in, buffer);
  if (!buffer.empty())
    molecule.setData("name", trimmed(buffer));

  // Atom count
  getline(in, buffer);
  buffer = trimmed(buffer);
  bool ok;
  size_t numAtoms = lexicalCast<size_t>(buffer, ok);
  if (buffer.empty() || !ok) {
    appendError("Number of atoms (line 2) invalid.");
    return false;
  }

  // read atom info:
  typedef map<string, unsigned char> AtomTypeMap;
  AtomTypeMap atomTypes;
  unsigned char customElementCounter = CustomElementMin;
  Vector3 pos;
  while (numAtoms-- > 0) {
    getline(in, buffer);
    // Figure out the distance between decimal points, implement support for
    // variable precision as specified:
    // "any number of decimal places, the format will then be n+5 positions with
    // n decimal places (n+1 for velocities) in stead of 8 with 3 (with 4 for
    // velocities)".
    size_t decimal1 = buffer.find(".", 20);
    size_t decimal2 = string::npos;
    int decimalSep = 0;
    if (decimal1 != string::npos)
      decimal2 = buffer.find(".", decimal1 + 1);
    if (decimal2 != string::npos)
      decimalSep = decimal2 - decimal1;
    if (decimalSep == 0) {
      appendError("Decimal separation of 0 found in atom positions: " + buffer);
      return false;
    }

    if (buffer.size() < static_cast<size_t>(20 + 3 * decimalSep)) {
      appendError("Error reading atom specification -- line too short: "
                  + buffer);
      return false;
    }

    // Format of buffer is: (all indices start at 1, variable dp throws this).
    // Offset:  0 format: %5i   value: Residue number
    // Offset:  5 format: %-5s  value: Residue name
    // Offset: 10 format: %5s   value: Atom name
    // Offset: 15 format: %5i   value: Atom number
    // Offset: 20 format: %8.3f value: x coordinate (nm)
    // Offset: 28 format: %8.3f value: y coordinate (nm)
    // Offset: 36 format: %8.3f value: z coordinate (nm)
    // Offset: 44 format: %8.4f value: x velocity (nm/ps, a.k.a. km/s)
    // Offset: 52 format: %8.4f value: y velocity (nm/ps, a.k.a. km/s)
    // Offset: 60 format: %8.4f value: z velocity (nm/ps, a.k.a. km/s)

    // Atom name:
    value = trimmed(buffer.substr(10, 5));
    AtomTypeMap::const_iterator it = atomTypes.find(value);
    if (it == atomTypes.end()) {
      atomTypes.insert(std::make_pair(value, customElementCounter++));
      it = atomTypes.find(value);
      if (customElementCounter > CustomElementMax) {
        appendError("Custom element type limit exceeded.");
        return false;
      }
    }
    Atom atom = molecule.addAtom(it->second);

    // Coords
    for (int i = 0; i < 3; ++i) {
      value = trimmed(buffer.substr(20 + i * decimalSep, decimalSep));
      pos[i] = lexicalCast<Real>(value, ok);
      if (!ok || value.empty()) {
        appendError("Error reading atom specification -- invalid coordinate: '"
                    + buffer + "' (bad coord: '" + value + "')");
        return false;
      }
    }
    atom.setPosition3d(pos * static_cast<Real>(10.0)); // nm --> Angstrom
  }

  // Set the custom element map if needed:
  if (!atomTypes.empty()) {
    Molecule::CustomElementMap elementMap;
    for (AtomTypeMap::const_iterator it = atomTypes.begin(),
         itEnd = atomTypes.end(); it != itEnd; ++it) {
      elementMap.insert(std::make_pair(it->second, it->first));
    }
    molecule.setCustomElementMap(elementMap);
  }

  // Box description:
  // v1(x) v2(y) v3(z) [v1(y) v1(z) v2(x) v2(z) v3(x) v3(y)]
  // The last six values may be omitted, set all non-specified values to 0.
  // v1(y) == v1(z) == v2(z) == 0 always.
  getline(in, buffer);
  vector<string> tokens(split(buffer, ' ', true));
  if (tokens.size() > 0) {
    if (tokens.size() != 3 && tokens.size() != 9) {
      appendError("Invalid box specification -- need either 3 or 9 values: '"
                  + buffer + "'");
      return false;
    }

    // Index arrays for parsing loop:
    const int rows[] = {0, 1, 2, 1, 2, 0, 2, 0, 1};
    const int cols[] = {0, 1, 2, 0, 0, 1, 1, 2, 2};

    Matrix3 cellMatrix = Matrix3::Zero();
    for (size_t i = 0; i < tokens.size(); ++i) {
      cellMatrix(rows[i], cols[i]) = lexicalCast<Real>(tokens[i], ok);
      if (!ok || tokens[i].empty()) {
        appendError("Invalid box specification -- bad value: '"
                    + tokens[i] + "'");
        return false;
      }
    }

    UnitCell *cell = new UnitCell;
    cell->setCellMatrix(cellMatrix * static_cast<Real>(10)); // nm --> Angstrom
    molecule.setUnitCell(cell);
  }

  return true;
}

bool GromacsFormat::write(std::ostream &, const Core::Molecule &)
{
  return false;
}

} // namespace Io
} // namespace Avogadro
<|endoftext|>


<commit_before>#include <stdio.h>
#include <sys/time.h>
#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <deque>
#include <unordered_map>
#include <algorithm>
#include <omp.h>

// Typedef for our primary data types
typedef std::vector<std::string> pages_t;
typedef std::unordered_map<std::string, int> page_ids_t;
typedef std::unordered_map<int,std::vector<int>> page_links_t;

/* Load in the page titles and page ids from a file.
 *
 * Each line of the file will represent the page ID, starting from line 1 until
 * the end of the file. The page titles will be stored into the pages vector.
 * The page titles => ids will be stored in the page_ids map
 *
 * Returns: The total number of pages loaded
 */
int load_page_titles(const std::string &file_path, pages_t &pages, page_ids_t &page_ids)
{
	std::ifstream page_titles_file(file_path);
	std::string page_name;

	// We won't be using the first index of the vector for anything since
	// the page_id indexes start at 1 instead of 0. This should help to
	// prevent any errors during development
	pages.push_back("");

	// Read the page name into the vector and into the hash-map
	int index = 1;

	while (std::getline(page_titles_file, page_name))
	{
		pages.push_back(page_name);
		page_ids[page_name] = index++;
	}

	return index - 1;
}

/* Load in the page links from a file.
 *
 * Each line of the file should start with a page_id that represents a page, it
 * should then follow with a colon (':'). All numbers after the colon represent
 * page_ids that that page links to.
 *
 * Read all page links into the page_links hash map, where the key is the
 * page_id and the value is a vector of page_ids the page links to.
 *
 * Returns: The total number of links loaded
 */
int load_page_links(const std::string &file_path, page_links_t &page_links)
{
	std::ifstream page_links_file(file_path);
	std::string link_id;

	int current_link = 0;
	int total_links  = 0;

	while (page_links_file >> link_id)
	{
		if (link_id.back() == ':')
		{
			current_link = atoi(link_id.c_str());
		}
		else
		{
			page_links[current_link].push_back(atoi(link_id.c_str()));
			total_links++;
		}
	}

	return total_links;
}

/* Given a hash map containing page_ids that represent a path back to a given
 * root node extract those Ids and order them so that we move from the root node
 * back to the tail_id.
 *
 * The root node is the node whos parent value is -1.
 *
 * Returns: A vector containing the page ids of the path in order.
 */
std::vector<int> path_from_parents(std::unordered_map<int, int> &parent_links,
	int target_id)
{
	int current = target_id;
	std::vector<int> target_path;

	// Only fill the target_path vector if we have a parent link chain
	if (parent_links[target_id])
	{
		// Traverse thrugh the parent_links till we reach the root node
		do
		{
			target_path.push_back(current);
		}
		while ((current = parent_links[current]) != -1);

		// Reverse the target path list to be in proper order
		std::reverse(target_path.begin(), target_path.end());
	}

	return target_path;
}

/* Calculate the shortest path between two page_ids in the page links graph.
 * This uses the Depth First Search algorithm to travers through each of the
 * child nodes looking to see if they are the target id. If none are, then we
 * will traverse through all their children, up until we can find the target id.
 *
 * Returns: See the return value of `path_from_parents`.
 */
std::vector<int> path_between_pages(page_links_t &page_links,
	int start_id, int target_id)
{
	// Keep track of parent nodes so we can traverse back through the path
	std::unordered_map<int, int> parent_links;

	// The starting node will always be considered 'visited'
	parent_links[start_id] = -1;

	// Queue of the current nodes to look at
	std::deque<int> current_nodes;
	current_nodes.push_back(start_id);

	bool found_target = false;

	// Search until we find our target node
	while ( ! found_target && ! current_nodes.empty())
	{
		int available_nodes = current_nodes.size();

		#pragma omp parallel for
		for (int i = 0; i < available_nodes; ++i)
		{
			// Since we can't break out of parallelize loops we just need to
			// finish off what evers left by continuing through them all
			#pragma omp flush (found_target)
			if (found_target) continue;

			int current;

			#pragma omp critical
			{
				current = current_nodes.front();
				current_nodes.pop_front();
			}

			// Iterate through all linked pages
			for (int page_id : page_links[current])
			{
				// Keep track of how we traveled to this node
				#pragma omp critical
				if (parent_links[page_id] == 0)
				{
					parent_links[page_id] = current;

					if (target_id == page_id)
					{
						found_target = true;
						#pragma omp flush (found_target)
					}
					else
					{
						current_nodes.push_back(page_id);
					}
				}
			}
		}
	}

	return path_from_parents(parent_links, target_id);
}

int main(int argc, char* argv[])
{
	if (argc != 5)
	{
		std::cerr << "Path Finder only accepts 4 arguments\n";
		exit(1);
	}

	// Used to store a list of page titles. Each index represents a page title
	pages_t pages;

	// Used to reverse lookup a page ID by it's title
	page_ids_t page_ids;

	// Used to store a list of page_ids that a given page_id links to
	page_links_t page_links;

	// Load in all page titles into a vector
	std::cout << "\033[92m==>\033[0m Reading in page titles as a vector & map\n";

	{
		struct timeval start, end;

		gettimeofday(&start, 0);
		int total = load_page_titles(std::string(argv[1]), pages, page_ids);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[94m  -> \033[0mRead in " << total    << " page titles\n";
		std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n";
	}

	// Load in all page links into a hash map
	std::cout << "\033[92m==>\033[0m Reading in page links as a map\n";

	{
		struct timeval start, end;

		gettimeofday(&start, 0);
		int total = load_page_links(std::string(argv[2]), page_links);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[94m  -> \033[0mRead in " << total    << " page links\n";
		std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n";
	}

	std::cout << "\033[92m==>\033[0m Beginning BFS on Wikipedia graph\n";

	// Get the ID's of the start and target nodes
	int start_id  = page_ids[argv[3]],
	    target_id = page_ids[argv[4]];

	if (start_id == target_id)
	{
		std::cout << "\n\033[92mNo clicks required. The page is the same!\n";
	}

	std::vector<int> target_path;

	struct timeval start, end;

	gettimeofday(&start, 0);
	target_path = path_between_pages(page_links, start_id, target_id);
	gettimeofday(&end, 0);

	double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
	std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n\n";

	// Print out the path between the pages
	for (int page_id : target_path)
	{
		std::cout << " * " << pages[page_id] << '\n';
	}

	return 0;
}
<commit_msg>Get data paths from relative bin location<commit_after>#include <stdio.h>
#include <unistd.h>
#include <libgen.h>
#include <sys/time.h>
#include <omp.h>

#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <deque>
#include <unordered_map>
#include <algorithm>

// Typedef for our primary data types
typedef std::vector<std::string> pages_t;
typedef std::unordered_map<std::string, int> page_ids_t;
typedef std::unordered_map<int,std::vector<int>> page_links_t;

/* Load in the page titles and page ids from a file.
 *
 * Each line of the file will represent the page ID, starting from line 1 until
 * the end of the file. The page titles will be stored into the pages vector.
 * The page titles => ids will be stored in the page_ids map
 *
 * Returns: The total number of pages loaded
 */
int load_page_titles(const std::string &file_path, pages_t &pages, page_ids_t &page_ids)
{
	std::ifstream page_titles_file(file_path);
	std::string page_name;

	// We won't be using the first index of the vector for anything since
	// the page_id indexes start at 1 instead of 0. This should help to
	// prevent any errors during development
	pages.push_back("");

	// Read the page name into the vector and into the hash-map
	int index = 1;

	while (std::getline(page_titles_file, page_name))
	{
		pages.push_back(page_name);
		page_ids[page_name] = index++;
	}

	return index - 1;
}

/* Load in the page links from a file.
 *
 * Each line of the file should start with a page_id that represents a page, it
 * should then follow with a colon (':'). All numbers after the colon represent
 * page_ids that that page links to.
 *
 * Read all page links into the page_links hash map, where the key is the
 * page_id and the value is a vector of page_ids the page links to.
 *
 * Returns: The total number of links loaded
 */
int load_page_links(const std::string &file_path, page_links_t &page_links)
{
	std::ifstream page_links_file(file_path);
	std::string link_id;

	int current_link = 0;
	int total_links  = 0;

	while (page_links_file >> link_id)
	{
		if (link_id.back() == ':')
		{
			current_link = atoi(link_id.c_str());
		}
		else
		{
			page_links[current_link].push_back(atoi(link_id.c_str()));
			total_links++;
		}
	}

	return total_links;
}

/* Given a hash map containing page_ids that represent a path back to a given
 * root node extract those Ids and order them so that we move from the root node
 * back to the tail_id.
 *
 * The root node is the node whos parent value is -1.
 *
 * Returns: A vector containing the page ids of the path in order.
 */
std::vector<int> path_from_parents(std::unordered_map<int, int> &parent_links,
	int target_id)
{
	int current = target_id;
	std::vector<int> target_path;

	// Only fill the target_path vector if we have a parent link chain
	if (parent_links[target_id])
	{
		// Traverse thrugh the parent_links till we reach the root node
		do
		{
			target_path.push_back(current);
		}
		while ((current = parent_links[current]) != -1);

		// Reverse the target path list to be in proper order
		std::reverse(target_path.begin(), target_path.end());
	}

	return target_path;
}

/* Calculate the shortest path between two page_ids in the page links graph.
 * This uses the Depth First Search algorithm to travers through each of the
 * child nodes looking to see if they are the target id. If none are, then we
 * will traverse through all their children, up until we can find the target id.
 *
 * Returns: See the return value of `path_from_parents`.
 */
std::vector<int> path_between_pages(page_links_t &page_links,
	int start_id, int target_id)
{
	// Keep track of parent nodes so we can traverse back through the path
	std::unordered_map<int, int> parent_links;

	// The starting node will always be considered 'visited'
	parent_links[start_id] = -1;

	// Queue of the current nodes to look at
	std::deque<int> current_nodes;
	current_nodes.push_back(start_id);

	bool found_target = false;

	// Search until we find our target node
	while ( ! found_target && ! current_nodes.empty())
	{
		int available_nodes = current_nodes.size();

		#pragma omp parallel for
		for (int i = 0; i < available_nodes; ++i)
		{
			// Since we can't break out of parallelize loops we just need to
			// finish off what evers left by continuing through them all
			#pragma omp flush (found_target)
			if (found_target) continue;

			int current;

			#pragma omp critical
			{
				current = current_nodes.front();
				current_nodes.pop_front();
			}

			// Iterate through all linked pages
			for (int page_id : page_links[current])
			{
				// Keep track of how we traveled to this node
				#pragma omp critical
				if (parent_links[page_id] == 0)
				{
					parent_links[page_id] = current;

					if (target_id == page_id)
					{
						found_target = true;
						#pragma omp flush (found_target)
					}
					else
					{
						current_nodes.push_back(page_id);
					}
				}
			}
		}
	}

	return path_from_parents(parent_links, target_id);
}

int main(int argc, char* argv[])
{
	// Used to store a list of page titles. Each index represents a page title
	pages_t pages;

	// Used to reverse lookup a page ID by it's title
	page_ids_t page_ids;

	// Used to store a list of page_ids that a given page_id links to
	page_links_t page_links;

	std::cout << "Welcome to Six-Degrees-of-Wikipedia path finder\n"
	          << "Please while we load Wikipedia into memory.\n\n";

	// Determine the base directory path
	char path_buffer[1024];
	readlink("/proc/self/exe", path_buffer, sizeof(path_buffer) - 1);
	std::string base_path = std::string(dirname(path_buffer)) + "/../";

	// Load in all page titles into a vector
	std::cout << "\033[92m==>\033[0m Reading in page titles as a vector & map\n";

	{
		struct timeval start, end;

		gettimeofday(&start, 0);
		int total = load_page_titles(base_path + "data/titles-sorted", pages, page_ids);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[94m  -> \033[0mRead in " << total    << " page titles\n";
		std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n";
	}

	// Load in all page links into a hash map
	std::cout << "\033[92m==>\033[0m Reading in page links as a map\n";

	{
		struct timeval start, end;

		gettimeofday(&start, 0);
		int total = load_page_links(base_path + "data/links-simple-sorted", page_links);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[94m  -> \033[0mRead in " << total    << " page links\n";
		std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n";
	}

	std::cout << "\033[92m==>\033[0m Beginning BFS on Wikipedia graph\n";

	// Get the ID's of the start and target nodes
	int start_id  = page_ids[argv[3]],
	    target_id = page_ids[argv[4]];

	if (start_id == target_id)
	{
		std::cout << "\n\033[92mNo clicks required. The page is the same!\n";
	}

	std::vector<int> target_path;

	struct timeval start, end;

	gettimeofday(&start, 0);
	target_path = path_between_pages(page_links, start_id, target_id);
	gettimeofday(&end, 0);

	double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
	std::cout << "\033[94m  -> \033[0mTook "    << duration << " seconds\n\n";

	// Print out the path between the pages
	for (int page_id : target_path)
	{
		std::cout << " * " << pages[page_id] << '\n';
	}

	return 0;
}
<|endoftext|>


<commit_before>#include <stdio.h>
#include <unistd.h>
#include <libgen.h>
#include <sys/time.h>
#include <omp.h>

#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <deque>
#include <unordered_map>
#include <algorithm>

// Typedef for our primary data types
typedef std::vector<std::string> pages_t;
typedef std::unordered_map<std::string, int> page_ids_t;
typedef std::unordered_map<int,std::vector<int>> page_links_t;

/* Load in the page titles and page ids from a file.
 *
 * Each line of the file will represent the page ID, starting from line 1 until
 * the end of the file. The page titles will be stored into the pages vector.
 * The page titles => ids will be stored in the page_ids map
 *
 * Returns: The total number of pages loaded
 */
int load_page_titles(const std::string &file_path, pages_t &pages, page_ids_t &page_ids)
{
	std::ifstream page_titles_file(file_path);
	std::string page_name;

	// We won't be using the first index of the vector for anything since
	// the page_id indexes start at 1 instead of 0. This should help to
	// prevent any errors during development
	pages.push_back("");

	// Read the page name into the vector and into the hash-map
	int index = 1;

	while (std::getline(page_titles_file, page_name))
	{
		pages.push_back(page_name);
		page_ids[page_name] = index++;
	}

	return index - 1;
}

/* Load in the page links from a file.
 *
 * Each line of the file should start with a page_id that represents a page, it
 * should then follow with a colon (':'). All numbers after the colon represent
 * page_ids that that page links to.
 *
 * Read all page links into the page_links hash map, where the key is the
 * page_id and the value is a vector of page_ids the page links to.
 *
 * Returns: The total number of links loaded
 */
int load_page_links(const std::string &file_path, page_links_t &page_links)
{
	std::ifstream page_links_file(file_path);
	std::string link_id;

	int current_link = 0;
	int total_links  = 0;

	while (page_links_file >> link_id)
	{
		if (link_id.back() == ':')
		{
			current_link = atoi(link_id.c_str());
		}
		else
		{
			page_links[current_link].push_back(atoi(link_id.c_str()));
			total_links++;
		}
	}

	return total_links;
}

/* Given a hash map containing page_ids that represent a path back to a given
 * root node extract those Ids and order them so that we move from the root node
 * back to the tail_id.
 *
 * The root node is the node whos parent value is -1.
 *
 * Returns: A vector containing the page ids of the path in order.
 */
std::vector<int> path_from_parents(std::unordered_map<int, int> &parent_links,
	int target_id)
{
	int current = target_id;
	std::vector<int> target_path;

	// Only fill the target_path vector if we have a parent link chain
	if (parent_links[target_id])
	{
		// Traverse thrugh the parent_links till we reach the root node
		do
		{
			target_path.push_back(current);
		}
		while ((current = parent_links[current]) != -1);

		// Reverse the target path list to be in proper order
		std::reverse(target_path.begin(), target_path.end());
	}

	return target_path;
}

/* Calculate the shortest path between two page_ids in the page links graph.
 * This uses the Depth First Search algorithm to travers through each of the
 * child nodes looking to see if they are the target id. If none are, then we
 * will traverse through all their children, up until we can find the target id.
 *
 * Returns: See the return value of `path_from_parents`.
 */
std::vector<int> path_between_pages(page_links_t &page_links,
	int start_id, int target_id)
{
	// Keep track of parent nodes so we can traverse back through the path
	std::unordered_map<int, int> parent_links;

	// The starting node will always be considered 'visited'
	parent_links[start_id] = -1;

	// Queue of the current nodes to look at
	std::deque<int> current_nodes;
	current_nodes.push_back(start_id);

	bool found_target = false;

	// Search until we find our target node
	while ( ! found_target && ! current_nodes.empty())
	{
		int available_nodes = current_nodes.size();

		#pragma omp parallel for
		for (int i = 0; i < available_nodes; ++i)
		{
			// Since we can't break out of parallelize loops we just need to
			// finish off what evers left by continuing through them all
			#pragma omp flush (found_target)
			if (found_target) continue;

			int current;

			#pragma omp critical
			{
				current = current_nodes.front();
				current_nodes.pop_front();
			}

			// Iterate through all linked pages
			for (int page_id : page_links[current])
			{
				// Keep track of how we traveled to this node
				#pragma omp critical
				if (parent_links[page_id] == 0)
				{
					parent_links[page_id] = current;

					if (target_id == page_id)
					{
						found_target = true;
						#pragma omp flush (found_target)
					}
					else
					{
						current_nodes.push_back(page_id);
					}
				}
			}
		}
	}

	return path_from_parents(parent_links, target_id);
}

/* Use the external python searcher script to find a given page by a search term
 * and output the resulting page name.
 *
 * Returns: Valid Wikipedia page name, or an empty string if one cannot be found
 */
std::string search_for_page(const std::string &py_searcher_bin, const std::string &search)
{
	FILE *pipe = popen((py_searcher_bin + " \"" + search + "\"").c_str(), "r");

	char buffer[255];
	std::string page_name = "";

	while ( ! feof(pipe))
	{
		if (fgets(buffer, 255, pipe) != NULL)
		{
			page_name += buffer;
		}
	}

	return page_name;
}

std::string pretty_print_path(const std::vector<int> &path_ids, const pages_t &pages)
{
	std::string path = "";
	int left_padding = 2;

	for (int page_id : path_ids)
	{
		std::string page_title = pages[page_id];
		std::string left_pad   = std::string(left_padding, ' ');

		path += left_pad + page_title + '\n';

		if (pages[page_id] != pages[path_ids.size() - 1])
		{
			path += left_pad + "↓\n";
		}
	}

	return path;
}

/* Kick off the main process:
 *
 *  1. Load in all of the wikipedia pages and page links into memory as a vector
 *     and hash map. Do this in parallel if possible
 *  2. Begin reading user input for what pages they would like to path find
 *  3. Use an external python script to get the page names from wikis search
 *  4. Path find using the BFS algorithm (again, in parallel)
 *  5. Repeat 2
 */
int main(int argc, char* argv[])
{
	// Used to store a list of page titles. Each index represents a page title
	pages_t pages;

	// Used to reverse lookup a page ID by it's title
	page_ids_t page_ids;

	// Used to store a list of page_ids that a given page_id links to
	page_links_t page_links;

	std::cout << "Welcome to Six-Degrees-of-Wikipedia path finder\n"
	          << "Please while we load Wikipedia into memory.\n\n";

	// Determine the base directory path
	char path_buffer[1024];
	readlink("/proc/self/exe", path_buffer, sizeof(path_buffer) - 1);
	std::string base_path = std::string(dirname(path_buffer)) + "/../";

	#pragma omp parallel sections
	{
		// Load in all data, in parallel
		std::cout << "\033[32m==>\033[0m Loading in page titles and page links\n";

		#pragma omp section
		{
			struct timeval start, end;

			gettimeofday(&start, 0);
			int total = load_page_titles(base_path + "data/titles-sorted", pages, page_ids);
			gettimeofday(&end, 0);

			double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
			std::cout << "\033[34m  -> \033[0mRead in " << total    << " page titles\n";
			std::cout << "\033[34m  -> \033[0mTook "    << duration << " seconds\n";
		}

		#pragma omp section
		{
			struct timeval start, end;

			gettimeofday(&start, 0);
			int total = load_page_links(base_path + "data/links-simple-sorted", page_links);
			gettimeofday(&end, 0);

			double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
			std::cout << "\033[34m  -> \033[0mRead in " << total    << " page links\n";
			std::cout << "\033[34m  -> \033[0mTook "    << duration << " seconds\n";
		}
	}

	std::cout << "\033[32m==>\033[0m Wikipedia graph loaded! Ready to path find!\n\n";

	while (true)
	{
		std::string start_page, target_page;

		std::cout << "Start Wikipedia Page: ";
		std::getline(std::cin, start_page);

		std::cout << "Target Wikipedia Page: ";
		std::getline(std::cin, target_page);

		// Use the the wiki_search.py script to lookup the wikipedia article name
		std::cout << "\n\033[32m==>\033[0m Looking up pages through Wikipedia search API\n";

		start_page  = search_for_page(base_path + "bin/wiki_search.py", start_page);
		target_page = search_for_page(base_path + "bin/wiki_search.py", target_page);

		// Ensure both pages exist
		if (start_page == "")
		{
			std::cout << "\033[31m==>\033[0m Start page is not a valid Wikipedia page\n\n";
			continue;
		}

		if (target_page == "")
		{
			std::cout << "\033[31m==>\033[0m Target page is not a valid Wikipedia page\n\n";
			continue;
		}

		// Remove newline from page names
		start_page.pop_back();
		target_page.pop_back();

		// Locate page Ids
		int start_id  = page_ids[start_page],
			target_id = page_ids[target_page];

		// Begin search
		std::cout << "\033[32m==>\033[0m Finding path from \033[35m"
		          << start_page  << "\033[0m → \033[34m"
		          << target_page << "\033[0m\n";

		std::vector<int> target_path;
		struct timeval start, end;

		gettimeofday(&start, 0);
		target_path = path_between_pages(page_links, start_id, target_id);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[34m  -> \033[0mFound path in \033[32m" << target_path.size() -1  << " clicks\033[0m\n";
		std::cout << "\033[34m  -> \033[0mTook " << duration << " seconds\n\n";

		// Pretty print the page path
		for (int page_id : target_path)
		{
			std::cout << "   \033[34m↓\033[0m " << pages[page_id] << "\n";
		}

		std::cout << '\n';
	}

	return 0;
}
<commit_msg>Time total boot process<commit_after>#include <stdio.h>
#include <unistd.h>
#include <libgen.h>
#include <sys/time.h>
#include <omp.h>

#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <deque>
#include <unordered_map>
#include <algorithm>

// Typedef for our primary data types
typedef std::vector<std::string> pages_t;
typedef std::unordered_map<std::string, int> page_ids_t;
typedef std::unordered_map<int,std::vector<int>> page_links_t;

/* Load in the page titles and page ids from a file.
 *
 * Each line of the file will represent the page ID, starting from line 1 until
 * the end of the file. The page titles will be stored into the pages vector.
 * The page titles => ids will be stored in the page_ids map
 *
 * Returns: The total number of pages loaded
 */
int load_page_titles(const std::string &file_path, pages_t &pages, page_ids_t &page_ids)
{
	std::ifstream page_titles_file(file_path);
	std::string page_name;

	// We won't be using the first index of the vector for anything since
	// the page_id indexes start at 1 instead of 0. This should help to
	// prevent any errors during development
	pages.push_back("");

	// Read the page name into the vector and into the hash-map
	int index = 1;

	while (std::getline(page_titles_file, page_name))
	{
		pages.push_back(page_name);
		page_ids[page_name] = index++;
	}

	return index - 1;
}

/* Load in the page links from a file.
 *
 * Each line of the file should start with a page_id that represents a page, it
 * should then follow with a colon (':'). All numbers after the colon represent
 * page_ids that that page links to.
 *
 * Read all page links into the page_links hash map, where the key is the
 * page_id and the value is a vector of page_ids the page links to.
 *
 * Returns: The total number of links loaded
 */
int load_page_links(const std::string &file_path, page_links_t &page_links)
{
	std::ifstream page_links_file(file_path);
	std::string link_id;

	int current_link = 0;
	int total_links  = 0;

	while (page_links_file >> link_id)
	{
		if (link_id.back() == ':')
		{
			current_link = atoi(link_id.c_str());
		}
		else
		{
			page_links[current_link].push_back(atoi(link_id.c_str()));
			total_links++;
		}
	}

	return total_links;
}

/* Given a hash map containing page_ids that represent a path back to a given
 * root node extract those Ids and order them so that we move from the root node
 * back to the tail_id.
 *
 * The root node is the node whos parent value is -1.
 *
 * Returns: A vector containing the page ids of the path in order.
 */
std::vector<int> path_from_parents(std::unordered_map<int, int> &parent_links,
	int target_id)
{
	int current = target_id;
	std::vector<int> target_path;

	// Only fill the target_path vector if we have a parent link chain
	if (parent_links[target_id])
	{
		// Traverse thrugh the parent_links till we reach the root node
		do
		{
			target_path.push_back(current);
		}
		while ((current = parent_links[current]) != -1);

		// Reverse the target path list to be in proper order
		std::reverse(target_path.begin(), target_path.end());
	}

	return target_path;
}

/* Calculate the shortest path between two page_ids in the page links graph.
 * This uses the Depth First Search algorithm to travers through each of the
 * child nodes looking to see if they are the target id. If none are, then we
 * will traverse through all their children, up until we can find the target id.
 *
 * Returns: See the return value of `path_from_parents`.
 */
std::vector<int> path_between_pages(page_links_t &page_links,
	int start_id, int target_id)
{
	// Keep track of parent nodes so we can traverse back through the path
	std::unordered_map<int, int> parent_links;

	// The starting node will always be considered 'visited'
	parent_links[start_id] = -1;

	// Queue of the current nodes to look at
	std::deque<int> current_nodes;
	current_nodes.push_back(start_id);

	bool found_target = false;

	// Search until we find our target node
	while ( ! found_target && ! current_nodes.empty())
	{
		int available_nodes = current_nodes.size();

		#pragma omp parallel for
		for (int i = 0; i < available_nodes; ++i)
		{
			// Since we can't break out of parallelize loops we just need to
			// finish off what evers left by continuing through them all
			#pragma omp flush (found_target)
			if (found_target) continue;

			int current;

			#pragma omp critical
			{
				current = current_nodes.front();
				current_nodes.pop_front();
			}

			// Iterate through all linked pages
			for (int page_id : page_links[current])
			{
				// Keep track of how we traveled to this node
				#pragma omp critical
				if (parent_links[page_id] == 0)
				{
					parent_links[page_id] = current;

					if (target_id == page_id)
					{
						found_target = true;
						#pragma omp flush (found_target)
					}
					else
					{
						current_nodes.push_back(page_id);
					}
				}
			}
		}
	}

	return path_from_parents(parent_links, target_id);
}

/* Use the external python searcher script to find a given page by a search term
 * and output the resulting page name.
 *
 * Returns: Valid Wikipedia page name, or an empty string if one cannot be found
 */
std::string search_for_page(const std::string &py_searcher_bin, const std::string &search)
{
	FILE *pipe = popen((py_searcher_bin + " \"" + search + "\"").c_str(), "r");

	char buffer[255];
	std::string page_name = "";

	while ( ! feof(pipe))
	{
		if (fgets(buffer, 255, pipe) != NULL)
		{
			page_name += buffer;
		}
	}

	return page_name;
}

std::string pretty_print_path(const std::vector<int> &path_ids, const pages_t &pages)
{
	std::string path = "";
	int left_padding = 2;

	for (int page_id : path_ids)
	{
		std::string page_title = pages[page_id];
		std::string left_pad   = std::string(left_padding, ' ');

		path += left_pad + page_title + '\n';

		if (pages[page_id] != pages[path_ids.size() - 1])
		{
			path += left_pad + "↓\n";
		}
	}

	return path;
}

/* Kick off the main process:
 *
 *  1. Load in all of the wikipedia pages and page links into memory as a vector
 *     and hash map. Do this in parallel if possible
 *  2. Begin reading user input for what pages they would like to path find
 *  3. Use an external python script to get the page names from wikis search
 *  4. Path find using the BFS algorithm (again, in parallel)
 *  5. Repeat 2
 */
int main(int argc, char* argv[])
{
	// Used to store a list of page titles. Each index represents a page title
	pages_t pages;

	// Used to reverse lookup a page ID by it's title
	page_ids_t page_ids;

	// Used to store a list of page_ids that a given page_id links to
	page_links_t page_links;

	std::cout << "Welcome to Six-Degrees-of-Wikipedia path finder\n"
	          << "Please while we load Wikipedia into memory.\n\n";

	// Determine the base directory path
	char path_buffer[1024];
	readlink("/proc/self/exe", path_buffer, sizeof(path_buffer) - 1);
	std::string base_path = std::string(dirname(path_buffer)) + "/../";

	struct timeval boot_start, boot_end;
	gettimeofday(&boot_start, 0);

	#pragma omp parallel sections
	{
		// Load in all data, in parallel
		std::cout << "\033[32m==>\033[0m Loading in page titles and page links\n";

		#pragma omp section
		{
			struct timeval start, end;

			gettimeofday(&start, 0);
			int total = load_page_titles(base_path + "data/titles-sorted", pages, page_ids);
			gettimeofday(&end, 0);

			double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
			std::cout << "\033[34m  -> \033[0mRead in " << total    << " page titles\n";
			std::cout << "\033[34m  -> \033[0mTook "    << duration << " seconds\n";
		}

		#pragma omp section
		{
			struct timeval start, end;

			gettimeofday(&start, 0);
			int total = load_page_links(base_path + "data/links-simple-sorted", page_links);
			gettimeofday(&end, 0);

			double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
			std::cout << "\033[34m  -> \033[0mRead in " << total    << " page links\n";
			std::cout << "\033[34m  -> \033[0mTook "    << duration << " seconds\n";
		}
	}

	gettimeofday(&boot_end, 0);
	double duration = ((boot_end.tv_sec - boot_start.tv_sec) * 1000000u
		+ boot_end.tv_usec - boot_start.tv_usec) / 1.e6;

	std::cout << "\033[32m==>\033[0m Wikipedia graph loaded in " << duration << " seconds. Ready to path find!\n\n";

	while (true)
	{
		std::string start_page, target_page;

		std::cout << "Start Wikipedia Page: ";
		std::getline(std::cin, start_page);

		std::cout << "Target Wikipedia Page: ";
		std::getline(std::cin, target_page);

		// Use the the wiki_search.py script to lookup the wikipedia article name
		std::cout << "\n\033[32m==>\033[0m Looking up pages through Wikipedia search API\n";

		start_page  = search_for_page(base_path + "bin/wiki_search.py", start_page);
		target_page = search_for_page(base_path + "bin/wiki_search.py", target_page);

		// Ensure both pages exist
		if (start_page == "")
		{
			std::cout << "\033[31m==>\033[0m Start page is not a valid Wikipedia page\n\n";
			continue;
		}

		if (target_page == "")
		{
			std::cout << "\033[31m==>\033[0m Target page is not a valid Wikipedia page\n\n";
			continue;
		}

		// Remove newline from page names
		start_page.pop_back();
		target_page.pop_back();

		// Locate page Ids
		int start_id  = page_ids[start_page],
			target_id = page_ids[target_page];

		// Begin search
		std::cout << "\033[32m==>\033[0m Finding path from \033[35m"
		          << start_page  << "\033[0m → \033[34m"
		          << target_page << "\033[0m\n";

		std::vector<int> target_path;
		struct timeval start, end;

		gettimeofday(&start, 0);
		target_path = path_between_pages(page_links, start_id, target_id);
		gettimeofday(&end, 0);

		double duration = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
		std::cout << "\033[34m  -> \033[0mFound path in \033[32m" << target_path.size() -1  << " clicks\033[0m\n";
		std::cout << "\033[34m  -> \033[0mTook " << duration << " seconds\n\n";

		// Pretty print the page path
		for (int page_id : target_path)
		{
			std::cout << "   \033[34m↓\033[0m " << pages[page_id] << "\n";
		}

		std::cout << '\n';
	}

	return 0;
}
<|endoftext|>


<commit_before>/*
  Copyright (C) 2015 Marcus Soll
  All rights reserved.

  You may use this file under the terms of BSD license as follows:

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the Jolla Ltd nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR
  ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "search.h"
#include <QDebug>

namespace {
void add_value(QString &s, QString value, QString key, QString &v_1, QString &v_2, QString &v_3, QString &v_4, QString &v_5, QString &v_6, QString &v_7, QString &v_8, int &count)
{
    ++count;
    switch(count)
    {
    case 1:
        v_1 = value;
        s.append(" WHERE ");
        s.append(key);
        break;
    case 2:
        v_2 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 3:
        v_3 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 4:
        v_4 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 5:
        v_5 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 6:
        v_6 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 7:
        v_7 = value;
        s.append(" AND ");
        s.append(key);
        break;
    case 8:
        v_8 = value;
        s.append(" AND ");
        s.append(key);
        break;
    default:
        qDebug() << "ERROR in " __FILE__ << " " << __LINE__ << ": Too many search terms";
    }
}
}

search::search(QString settings_path, QObject *parent) :
    QObject(parent),
    _database(),
    _kanji_query(),
    _settings_query(),
    _literal_result(""),
    _meaning_result(""),
    _saved(""),
    _literal(""),
    _radical(-1),
    _strokecount(-1),
    _jlpt(-1),
    _meaning(""),
    _search_for_saved(false),
    _saved_search_value(false),
    _skip1(0),
    _skip2(0),
    _skip3(0),
    _search_started(false)
{
    _database = QSqlDatabase::addDatabase("QSQLITE");
    _database.setDatabaseName(":memory:");
    if(!_database.open())
    {
        qCritical() << "ERROR in " __FILE__ << " " << __LINE__ << ": Can not create database";
    }
    _kanji_query = QSqlQuery(_database);
    _settings_query = QSqlQuery(_database);

    QString s = QString("ATTACH DATABASE '/usr/share/harbour-kanji/kanjidb.sqlite3' AS kanjidb");
    if(!_kanji_query.exec(s))
    {
        QString error = s.append(": ").append(_kanji_query.lastError().text());
        qWarning() << error;
    }
    s = QString("ATTACH DATABASE ? AS settingsdb");
    _kanji_query.clear();
    _kanji_query.prepare(s);
    _kanji_query.addBindValue(settings_path);
    if(!_kanji_query.exec())
    {
        QString error = s.append(": ").append(_kanji_query.lastError().text());
        qWarning() << error;
    }
    _kanji_query.clear();
}

void search::clear()
{
    _kanji_query.clear();
    _settings_query.clear();
    _literal_result = "";
    _meaning_result = "";
    _saved = false;
    _literal = "";
    _radical = -1;
    _strokecount = -1;
    _jlpt = -1;
    _meaning = "";
    _search_for_saved = false;
    _saved_search_value = false;
    _skip1 = 0;
    _skip2 = 0;
    _skip3 = 0;
    _search_started = false;
}

void search::search_literal(QString literal)
{
    _literal = literal;
}

void search::search_radical(int radical)
{
    _radical = radical;
}

void search::search_strokecount(int strokecount)
{
    _strokecount = strokecount;
}

void search::search_jlpt(int jlpt)
{
    _jlpt = jlpt;
}

void search::search_meaning(QString meaning)
{
    _meaning = meaning;
}

void search::search_saved(bool saved)
{
    _search_for_saved = true;
    _saved_search_value = saved;
}

void search::search_skip(int skip1, int skip2, int skip3)
{
    _skip1 = skip1;
    _skip2 = skip2;
    _skip3 = skip3;
}

bool search::start_search()
{
    if(_literal == "" && _radical == 0 && _strokecount == 0 && _jlpt == 0 && _meaning == "")
    {
        // Get all kanji
        QString s = QString("SELECT literal,meaning FROM kanjidb.kanji");
        if(!_kanji_query.exec(s))
        {
            QString error = s.append(": ").append(_kanji_query.lastError().text());
            qWarning() << error;
            _search_started = false;
            return false;
        }
        if(!_kanji_query.isSelect())
        {
            QString error = s.append(": No SELECT");
            qWarning() << error;
            _search_started = false;
            return false;
        }
    }
    else
    {
        // Search for all fitting kanji
        int count = 0;

        QString v_1;
        QString v_2;
        QString v_3;
        QString v_4;
        QString v_5;
        QString v_6;
        QString v_7;
        QString v_8;

        QString s = QString("SELECT literal,meaning FROM kanjidb.kanji");
        if(_literal != "")
        {
            add_value(s,QString("\%%1\%").arg(_literal),QString("literal LIKE ?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_radical != -1)
        {
            add_value(s,QString("%1").arg(_radical),QString("radical=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_strokecount != -1)
        {
            add_value(s,QString("%1").arg(_strokecount),QString("strokecount=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_jlpt != -1)
        {
            add_value(s,QString("%1").arg(_jlpt),QString("JLPT=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_meaning != "")
        {
            add_value(s,QString("\%%1\%").arg(_meaning),QString("meaning LIKE ?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_skip1 != 0)
        {
            add_value(s,QString("%1").arg(_skip1),QString("skip_1=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_skip2 != 0)
        {
            add_value(s,QString("%1").arg(_skip2),QString("skip_2=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }
        if(_skip3 != 0)
        {
            add_value(s,QString("%1").arg(_skip3),QString("skip_3=?"),v_1,v_2,v_3,v_4,v_5,v_6,v_7,v_8,count);
        }

        _kanji_query.clear();
        _kanji_query.prepare(s);

        if(count >= 1)
        {
            _kanji_query.addBindValue(v_1);
        }
        if(count >= 2)
        {
            _kanji_query.addBindValue(v_2);
        }
        if(count >= 3)
        {
            _kanji_query.addBindValue(v_3);
        }
        if(count >= 4)
        {
            _kanji_query.addBindValue(v_4);
        }
        if(count >= 5)
        {
            _kanji_query.addBindValue(v_5);
        }
        if(count >= 6)
        {
            _kanji_query.addBindValue(v_6);
        }
        if(count >= 7)
        {
            _kanji_query.addBindValue(v_7);
        }
        if(count >= 8)
        {
            _kanji_query.addBindValue(v_8);
        }

        qDebug() << "DEBUG in " __FILE__ << " " << __LINE__ << ": SELECT statement:" << s;

        if(!_kanji_query.exec())
        {
            QString error = s.append(": ").append(_kanji_query.lastError().text());
            qWarning() << error;
            _search_started = false;
            return false;
        }
        if(!_kanji_query.isSelect())
        {
            QString error = s.append(": No SELECT");
            qWarning() << error;
            _search_started = false;
            return false;
        }
    }
    _search_started = true;
    return true;
}

bool search::next_hidden()
{
    if(!_search_started)
    {
        return false;
    }
    if(_kanji_query.next())
    {
        _literal_result = _kanji_query.value(0).toString();
        _meaning_result = _kanji_query.value(1).toString();

        QString s = QString("SELECT count(*) FROM settingsdb.saved_kanji WHERE literal=?");
        _settings_query.prepare(s);
        _settings_query.addBindValue(_literal_result);
        if(_settings_query.exec() && _settings_query.isSelect() && _settings_query.next() && _settings_query.value(0).toInt() > 0)
        {
            _saved = true;
        }
        else
        {
            _saved = false;
        }
        _settings_query.finish();
        return true;
    }
    else
    {
        _search_started = false;
        return false;
    }
}

bool search::next()
{
    if(!_search_started)
    {
        return false;
    }
    if(!_search_for_saved)
    {
        return next_hidden();
    }
    else
    {
        while(next_hidden())
        {
            if(_saved_search_value == _saved)
            {
                return true;
            }
        }
        return false;
    }
}

QString search::literal()
{
    return _literal_result;
}

QString search::meaning()
{
    return _meaning_result;
}

bool search::kanji_is_saved()
{
    return _saved;
}
<commit_msg>If "meaning" is selected for search both "meaning" and "custom_translation" will be considered.<commit_after>/*
  Copyright (C) 2015 Marcus Soll
  All rights reserved.

  You may use this file under the terms of BSD license as follows:

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the Jolla Ltd nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR
  ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "search.h"
#include <QDebug>
#include <QVariant>
#include <QVariantList>

namespace {
QString get_seperator(int &count)
{
    ++count;
    if(count == 1)
    {
        return " WHERE ";
    }
    else
    {
        return " AND ";
    }
}
}

search::search(QString settings_path, QObject *parent) :
    QObject(parent),
    _database(),
    _kanji_query(),
    _settings_query(),
    _literal_result(""),
    _meaning_result(""),
    _saved(""),
    _literal(""),
    _radical(-1),
    _strokecount(-1),
    _jlpt(-1),
    _meaning(""),
    _search_for_saved(false),
    _saved_search_value(false),
    _skip1(0),
    _skip2(0),
    _skip3(0),
    _search_started(false)
{
    _database = QSqlDatabase::addDatabase("QSQLITE");
    _database.setDatabaseName(":memory:");
    if(!_database.open())
    {
        qCritical() << "ERROR in " __FILE__ << " " << __LINE__ << ": Can not create database";
    }
    _kanji_query = QSqlQuery(_database);
    _settings_query = QSqlQuery(_database);

    QString s = QString("ATTACH DATABASE '/usr/share/harbour-kanji/kanjidb.sqlite3' AS kanjidb");
    if(!_kanji_query.exec(s))
    {
        QString error = s.append(": ").append(_kanji_query.lastError().text());
        qWarning() << error;
    }
    s = QString("ATTACH DATABASE ? AS settingsdb");
    _kanji_query.clear();
    _kanji_query.prepare(s);
    _kanji_query.addBindValue(settings_path);
    if(!_kanji_query.exec())
    {
        QString error = s.append(": ").append(_kanji_query.lastError().text());
        qWarning() << error;
    }
    _kanji_query.clear();
}

void search::clear()
{
    _kanji_query.clear();
    _settings_query.clear();
    _literal_result = "";
    _meaning_result = "";
    _saved = false;
    _literal = "";
    _radical = -1;
    _strokecount = -1;
    _jlpt = -1;
    _meaning = "";
    _search_for_saved = false;
    _saved_search_value = false;
    _skip1 = 0;
    _skip2 = 0;
    _skip3 = 0;
    _search_started = false;
}

void search::search_literal(QString literal)
{
    _literal = literal;
}

void search::search_radical(int radical)
{
    _radical = radical;
}

void search::search_strokecount(int strokecount)
{
    _strokecount = strokecount;
}

void search::search_jlpt(int jlpt)
{
    _jlpt = jlpt;
}

void search::search_meaning(QString meaning)
{
    _meaning = meaning;
}

void search::search_saved(bool saved)
{
    _search_for_saved = true;
    _saved_search_value = saved;
}

void search::search_skip(int skip1, int skip2, int skip3)
{
    _skip1 = skip1;
    _skip2 = skip2;
    _skip3 = skip3;
}

bool search::start_search()
{
    if(_literal == "" && _radical == -1 && _strokecount == -1 && _jlpt == -1 && _meaning == "" && _skip1 == 0 && _skip2 == 0 && _skip3 == 0)
    {
        // Get all kanji
        QString s = QString("SELECT literal,meaning FROM kanjidb.kanji");
        if(!_kanji_query.exec(s))
        {
            QString error = s.append(": ").append(_kanji_query.lastError().text());
            qWarning() << error;
            _search_started = false;
            return false;
        }
        if(!_kanji_query.isSelect())
        {
            QString error = s.append(": No SELECT");
            qWarning() << error;
            _search_started = false;
            return false;
        }
    }
    else
    {
        // Search for all fitting kanji
        int count = 0;
        QVariantList search_list;

        QString s = QString("SELECT kanjidb.kanji.literal,meaning FROM kanjidb.kanji LEFT JOIN settingsdb.custom_translation ON kanjidb.kanji.literal=settingsdb.custom_translation.literal");
        if(_literal != "")
        {
            s.append(get_seperator(count));
            s.append("literal LIKE ?");
            search_list.append(QString("\%%1\%").arg(_literal));
        }
        if(_radical != -1)
        {
            s.append(get_seperator(count));
            s.append("radical=?");
            search_list.append(_radical);
        }
        if(_strokecount != -1)
        {
            s.append(get_seperator(count));
            s.append("strokecount=?");
            search_list.append(_strokecount);
        }
        if(_jlpt != -1)
        {
            s.append(get_seperator(count));
            s.append("JLPT=?");
            search_list.append(_jlpt);
        }
        if(_meaning != "")
        {
            s.append(get_seperator(count));
            s.append("(meaning LIKE ? OR translation LIKE ?)");
            search_list.append(QString("\%%1\%").arg(_meaning));
            search_list.append(QString("\%%1\%").arg(_meaning));
        }
        if(_skip1 != 0)
        {
            s.append(get_seperator(count));
            s.append("skip_1=?");
            search_list.append(_skip1);
        }
        if(_skip2 != 0)
        {
            s.append(get_seperator(count));
            s.append("skip_2=?");
            search_list.append(_skip2);
        }
        if(_skip3 != 0)
        {
            s.append(get_seperator(count));
            s.append("skip_3=?");
            search_list.append(_skip3);
        }

        _kanji_query.clear();
        _kanji_query.prepare(s);
        foreach (QVariant var, search_list) {
            _kanji_query.addBindValue(var);
        }

        qDebug() << "DEBUG in " __FILE__ << " " << __LINE__ << ": SELECT statement:" << s;

        if(!_kanji_query.exec())
        {
            QString error = s.append(": ").append(_kanji_query.lastError().text());
            qWarning() << error;
            _search_started = false;
            return false;
        }
        if(!_kanji_query.isSelect())
        {
            QString error = s.append(": No SELECT");
            qWarning() << error;
            _search_started = false;
            return false;
        }
    }
    _search_started = true;
    return true;
}

bool search::next_hidden()
{
    if(!_search_started)
    {
        return false;
    }
    if(_kanji_query.next())
    {
        _literal_result = _kanji_query.value(0).toString();
        _meaning_result = _kanji_query.value(1).toString();

        QString s = QString("SELECT count(*) FROM settingsdb.saved_kanji WHERE literal=?");
        _settings_query.prepare(s);
        _settings_query.addBindValue(_literal_result);
        if(_settings_query.exec() && _settings_query.isSelect() && _settings_query.next() && _settings_query.value(0).toInt() > 0)
        {
            _saved = true;
        }
        else
        {
            _saved = false;
        }
        _settings_query.finish();
        return true;
    }
    else
    {
        _search_started = false;
        return false;
    }
}

bool search::next()
{
    if(!_search_started)
    {
        return false;
    }
    if(!_search_for_saved)
    {
        return next_hidden();
    }
    else
    {
        while(next_hidden())
        {
            if(_saved_search_value == _saved)
            {
                return true;
            }
        }
        return false;
    }
}

QString search::literal()
{
    return _literal_result;
}

QString search::meaning()
{
    return _meaning_result;
}

bool search::kanji_is_saved()
{
    return _saved;
}
<|endoftext|>


<commit_before>/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#ifdef PADDLE_WITH_XPU

#include "paddle/fluid/operators/sequence_ops/sequence_conv_op.h"
#include "paddle/fluid/platform/device/device_wrapper.h"

namespace paddle {
namespace operators {
using Tensor = framework::Tensor;

template <typename DeviceContext, typename T>
class SequenceConvXPUKernel : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext& context) const override {
    auto* in = context.Input<LoDTensor>("X");
    auto* out = context.Output<LoDTensor>("Out");
    auto filter = *context.Input<Tensor>("Filter");

    out->mutable_data<T>(context.GetPlace());

    int context_start = context.Attr<int>("contextStart");
    int context_length = context.Attr<int>("contextLength");
    int context_stride = context.Attr<int>("contextStride");
    bool padding_trainable = context.Attr<bool>("paddingTrainable");

    PADDLE_ENFORCE_EQ(
        in->lod().empty(), false,
        platform::errors::InvalidArgument("Input(X) Tensor of SequenceConvOp "
                                          "does not contain LoD information."));
    PADDLE_ENFORCE_EQ(
        in->lod().size(), 1UL,
        platform::errors::InvalidArgument(
            "Only support input sequence with lod level equal to 1 at "
            "present. But received: lod level %u.",
            in->lod().size()));

    PADDLE_ENFORCE_EQ(
        padding_trainable, false,
        platform::errors::InvalidArgument("Only support padding_trainable "
                                          "equal false."));

    int up_pad = std::max(0, -context_start);
    int down_pad = std::max(0, context_start + context_length - 1);
    PADDLE_ENFORCE_EQ(up_pad, 2, platform::errors::InvalidArgument(
                                     "Only support up_pad equal 2."));
    PADDLE_ENFORCE_EQ(down_pad, 2, platform::errors::InvalidArgument(
                                       "Only support down_pad equal 2."));

    auto xpu_context =
        context.template device_context<DeviceContext>().x_context();
    auto sequence_width = static_cast<int64_t>(in->dims()[1]);
    framework::DDim col_shape = {in->dims()[0],
                                 context_length * sequence_width};
    xpu::ctx_guard RAII_GUARD(xpu_context);
    int col_numel = col_shape[0] * col_shape[1];
    T* col_data = RAII_GUARD.alloc_l3_or_gm<T>(col_numel);
    PADDLE_ENFORCE_NOT_NULL(
        col_data, paddle::platform::errors::Fatal("XPU memory is not enough"));

    auto lod_level_0 = in->lod()[0];
    int lod_size = lod_level_0.size();
    // If batch size set to 256, the lod is {0, batch[0] - 0,
    // batch[1] - batch [0], ..., batch[255] - batch[254]},
    // so the lod_size will be 257.
    PADDLE_ENFORCE_LE(lod_size, 257, platform::errors::InvalidArgument(
                                         "Only support batch size <= 256."));

    std::vector<int> cpu_lodx(lod_size);
    for (int i = 0; i < lod_size; i++) {
      cpu_lodx[i] = lod_level_0[i];
    }
    xpu::VectorParam<int> lodx = {cpu_lodx.data(),
                                  static_cast<int>(cpu_lodx.size()), nullptr};

    int r = xpu::sequence_context_projection<T, int>(
        xpu_context, in->data<T>(), col_data, nullptr, lodx, sequence_width,
        context_start, context_length, context_stride, {2, 2});
    PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection");

    bool trans_a = false;
    bool trans_b = false;
    int m = col_shape[0];
    int k = col_shape[1];
    int k1 = filter.dims()[0];
    int n = filter.dims()[1];
    PADDLE_ENFORCE_EQ(k, k1,
                      platform::errors::InvalidArgument(
                          "The shape of FC in SequenceConvOp is invalid."
                          "The k of matrix A is %d, k1 of matrix B is %d."
                          "But expect k == k1",
                          k, k1));
    int lda = (!trans_a) ? k : m;
    int ldb = (!trans_b) ? n : k;
    int ldc = n;
    T alpha = static_cast<T>(1.0);
    T beta = static_cast<T>(0.0);
    const T* data_a = col_data;
    const T* data_b = filter.data<T>();
    T* data_c = out->data<T>();

    r = xpu::fc_fusion<T, T, T, int32_t>(
        xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b, nullptr,
        nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
        xpu::Activation_t::LINEAR);
    PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
    if (xpu_context->xpu_stream != nullptr) {
      xpu_wait(xpu_context->xpu_stream);
    }
  }
};

template <typename DeviceContext, typename T>
class SequenceConvGradXPUKernel : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext& context) const override {
    auto* in_g = context.Output<LoDTensor>(framework::GradVarName("X"));
    auto* out_g = context.Input<LoDTensor>(framework::GradVarName("Out"));
    auto* filter_g = context.Output<Tensor>(framework::GradVarName("Filter"));
    auto* in = context.Input<LoDTensor>("X");
    auto* filter = context.Input<Tensor>("Filter");

    int context_start = context.Attr<int>("contextStart");
    int context_length = context.Attr<int>("contextLength");
    int context_stride = context.Attr<int>("contextStride");
    bool padding_trainable = context.Attr<bool>("paddingTrainable");

    PADDLE_ENFORCE_EQ(
        in->lod().empty(), false,
        platform::errors::InvalidArgument("Input(X) Tensor of SequenceConvOp "
                                          "does not contain LoD information."));
    PADDLE_ENFORCE_EQ(
        in->lod().size(), 1UL,
        platform::errors::InvalidArgument(
            "Only support input sequence with lod level equal to 1 at "
            "present. But received: lod level %u.",
            in->lod().size()));

    PADDLE_ENFORCE_EQ(
        padding_trainable, false,
        platform::errors::InvalidArgument("Only support padding_trainable "
                                          "equal false."));

    int up_pad = std::max(0, -context_start);
    int down_pad = std::max(0, context_start + context_length - 1);
    PADDLE_ENFORCE_EQ(up_pad, 2, platform::errors::InvalidArgument(
                                     "Only support up_pad equal 2."));
    PADDLE_ENFORCE_EQ(down_pad, 2, platform::errors::InvalidArgument(
                                       "Only support down_pad equal 2."));

    auto lod_level_0 = in->lod()[0];
    int lod_size = lod_level_0.size();
    PADDLE_ENFORCE_LE(lod_size, 257, platform::errors::InvalidArgument(
                                         "Only support batch size <= 256."));

    std::vector<int> cpu_lodx(lod_size);
    for (int i = 0; i < lod_size; i++) {
      cpu_lodx[i] = lod_level_0[i];
    }
    xpu::VectorParam<int> lodx = {cpu_lodx.data(),
                                  static_cast<int>(cpu_lodx.size()), nullptr};

    auto xpu_context =
        context.template device_context<DeviceContext>().x_context();
    auto sequence_width = static_cast<int64_t>(in->dims()[1]);
    framework::DDim col_shape = {in->dims()[0],
                                 context_length * sequence_width};
    xpu::ctx_guard RAII_GUARD(xpu_context);
    int col_numel = col_shape[0] * col_shape[1];
    T* col_data = RAII_GUARD.alloc_l3_or_gm<T>(col_numel);
    PADDLE_ENFORCE_NOT_NULL(
        col_data, paddle::platform::errors::Fatal("XPU memory is not enough"));

    if (in_g || filter_g) {
      int r = xpu::constant<T>(xpu_context, col_data, col_numel, T(0));
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "constant");

      bool trans_a = false;
      bool trans_b = true;
      int m = out_g->dims()[0];
      int k = out_g->dims()[1];
      int n = filter->dims()[0];
      int k1 = filter->dims()[1];
      PADDLE_ENFORCE_EQ(k, k1,
                        platform::errors::InvalidArgument(
                            "The shape of FC in SequenceConvGradOp is invalid."
                            "The k of matrix A is %d, k1 of matrix B is %d."
                            "But expect k == k1",
                            k, k1));
      int lda = (!trans_a) ? k : m;
      int ldb = (!trans_b) ? n : k;
      int ldc = n;
      T alpha = static_cast<T>(1.0);
      T beta = static_cast<T>(0.0);
      const T* data_a = out_g->data<T>();
      const T* data_b = filter->data<T>();
      T* data_c = col_data;

      r = xpu::fc_fusion<T, T, T, int32_t>(
          xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b,
          nullptr, nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
          xpu::Activation_t::LINEAR);
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
    }

    if (in_g) {
      PADDLE_ENFORCE_LT(sequence_width, 512,
                        platform::errors::InvalidArgument(
                            "Only support sequence_width < 512."));

      in_g->mutable_data<T>(context.GetPlace());
      in_g->set_lod(in->lod());
      xpu::constant<T>(xpu_context, in_g->data<T>(), in_g->numel(), T(0));

      int r = xpu::sequence_context_projection_grad<T, int>(
          xpu_context, in_g->data<T>(), col_data, nullptr, lodx, sequence_width,
          context_start, context_length, context_stride, {2, 2});
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection_grad");
    }

    if (filter_g) {
      filter_g->mutable_data<T>(context.GetPlace());
      xpu::constant<T>(xpu_context, filter_g->data<T>(), filter_g->numel(),
                       T(0));

      int r = xpu::sequence_context_projection<T, int>(
          xpu_context, in->data<T>(), col_data, nullptr, lodx, sequence_width,
          context_start, context_length, context_stride, {2, 2});
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection");

      bool trans_a = true;
      bool trans_b = false;
      int k = col_shape[0];
      int m = col_shape[1];
      int k1 = out_g->dims()[0];
      int n = out_g->dims()[1];
      PADDLE_ENFORCE_EQ(k, k1,
                        platform::errors::InvalidArgument(
                            "The shape of FC in SequenceConvGradOp is invalid."
                            "The k of matrix A is %d, k1 of matrix B is %d."
                            "But expect k == k1",
                            k, k1));
      int lda = (!trans_a) ? k : m;
      int ldb = (!trans_b) ? n : k;
      int ldc = n;
      T alpha = static_cast<T>(1.0);
      T beta = static_cast<T>(0.0);
      const T* data_a = col_data;
      const T* data_b = out_g->data<T>();
      T* data_c = filter_g->data<T>();

      r = xpu::fc_fusion<T, T, T, int32_t>(
          xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b,
          nullptr, nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
          xpu::Activation_t::LINEAR);
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
      if (xpu_context->xpu_stream != nullptr) {
        xpu_wait(xpu_context->xpu_stream);
      }
    }
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP_XPU_KERNEL(
    sequence_conv,
    ops::SequenceConvXPUKernel<paddle::platform::XPUDeviceContext, float>);

REGISTER_OP_XPU_KERNEL(
    sequence_conv_grad,
    ops::SequenceConvGradXPUKernel<paddle::platform::XPUDeviceContext, float>);

#endif
<commit_msg>remove unnecessary constant fill in sequence conv test=kunlun. (#40126)<commit_after>/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#ifdef PADDLE_WITH_XPU

#include "paddle/fluid/operators/sequence_ops/sequence_conv_op.h"
#include "paddle/fluid/platform/device/device_wrapper.h"

namespace paddle {
namespace operators {
using Tensor = framework::Tensor;

template <typename DeviceContext, typename T>
class SequenceConvXPUKernel : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext& context) const override {
    auto* in = context.Input<LoDTensor>("X");
    auto* out = context.Output<LoDTensor>("Out");
    auto filter = *context.Input<Tensor>("Filter");

    out->mutable_data<T>(context.GetPlace());

    int context_start = context.Attr<int>("contextStart");
    int context_length = context.Attr<int>("contextLength");
    int context_stride = context.Attr<int>("contextStride");
    bool padding_trainable = context.Attr<bool>("paddingTrainable");

    PADDLE_ENFORCE_EQ(
        in->lod().empty(), false,
        platform::errors::InvalidArgument("Input(X) Tensor of SequenceConvOp "
                                          "does not contain LoD information."));
    PADDLE_ENFORCE_EQ(
        in->lod().size(), 1UL,
        platform::errors::InvalidArgument(
            "Only support input sequence with lod level equal to 1 at "
            "present. But received: lod level %u.",
            in->lod().size()));

    PADDLE_ENFORCE_EQ(
        padding_trainable, false,
        platform::errors::InvalidArgument("Only support padding_trainable "
                                          "equal false."));

    int up_pad = std::max(0, -context_start);
    int down_pad = std::max(0, context_start + context_length - 1);
    PADDLE_ENFORCE_EQ(up_pad, 2, platform::errors::InvalidArgument(
                                     "Only support up_pad equal 2."));
    PADDLE_ENFORCE_EQ(down_pad, 2, platform::errors::InvalidArgument(
                                       "Only support down_pad equal 2."));

    auto xpu_context =
        context.template device_context<DeviceContext>().x_context();
    auto sequence_width = static_cast<int64_t>(in->dims()[1]);
    framework::DDim col_shape = {in->dims()[0],
                                 context_length * sequence_width};
    xpu::ctx_guard RAII_GUARD(xpu_context);
    int col_numel = col_shape[0] * col_shape[1];
    T* col_data = RAII_GUARD.alloc_l3_or_gm<T>(col_numel);
    PADDLE_ENFORCE_NOT_NULL(
        col_data, paddle::platform::errors::Fatal("XPU memory is not enough"));

    auto lod_level_0 = in->lod()[0];
    int lod_size = lod_level_0.size();
    // If batch size set to 256, the lod is {0, batch[0] - 0,
    // batch[1] - batch [0], ..., batch[255] - batch[254]},
    // so the lod_size will be 257.
    PADDLE_ENFORCE_LE(lod_size, 257, platform::errors::InvalidArgument(
                                         "Only support batch size <= 256."));

    std::vector<int> cpu_lodx(lod_size);
    for (int i = 0; i < lod_size; i++) {
      cpu_lodx[i] = lod_level_0[i];
    }
    xpu::VectorParam<int> lodx = {cpu_lodx.data(),
                                  static_cast<int>(cpu_lodx.size()), nullptr};

    int r = xpu::sequence_context_projection<T, int>(
        xpu_context, in->data<T>(), col_data, nullptr, lodx, sequence_width,
        context_start, context_length, context_stride, {2, 2});
    PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection");

    bool trans_a = false;
    bool trans_b = false;
    int m = col_shape[0];
    int k = col_shape[1];
    int k1 = filter.dims()[0];
    int n = filter.dims()[1];
    PADDLE_ENFORCE_EQ(k, k1,
                      platform::errors::InvalidArgument(
                          "The shape of FC in SequenceConvOp is invalid."
                          "The k of matrix A is %d, k1 of matrix B is %d."
                          "But expect k == k1",
                          k, k1));
    int lda = (!trans_a) ? k : m;
    int ldb = (!trans_b) ? n : k;
    int ldc = n;
    T alpha = static_cast<T>(1.0);
    T beta = static_cast<T>(0.0);
    const T* data_a = col_data;
    const T* data_b = filter.data<T>();
    T* data_c = out->data<T>();

    r = xpu::fc_fusion<T, T, T, int32_t>(
        xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b, nullptr,
        nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
        xpu::Activation_t::LINEAR);
    PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
    if (xpu_context->xpu_stream != nullptr) {
      xpu_wait(xpu_context->xpu_stream);
    }
  }
};

template <typename DeviceContext, typename T>
class SequenceConvGradXPUKernel : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext& context) const override {
    auto* in_g = context.Output<LoDTensor>(framework::GradVarName("X"));
    auto* out_g = context.Input<LoDTensor>(framework::GradVarName("Out"));
    auto* filter_g = context.Output<Tensor>(framework::GradVarName("Filter"));
    auto* in = context.Input<LoDTensor>("X");
    auto* filter = context.Input<Tensor>("Filter");

    int context_start = context.Attr<int>("contextStart");
    int context_length = context.Attr<int>("contextLength");
    int context_stride = context.Attr<int>("contextStride");
    bool padding_trainable = context.Attr<bool>("paddingTrainable");

    PADDLE_ENFORCE_EQ(
        in->lod().empty(), false,
        platform::errors::InvalidArgument("Input(X) Tensor of SequenceConvOp "
                                          "does not contain LoD information."));
    PADDLE_ENFORCE_EQ(
        in->lod().size(), 1UL,
        platform::errors::InvalidArgument(
            "Only support input sequence with lod level equal to 1 at "
            "present. But received: lod level %u.",
            in->lod().size()));

    PADDLE_ENFORCE_EQ(
        padding_trainable, false,
        platform::errors::InvalidArgument("Only support padding_trainable "
                                          "equal false."));

    int up_pad = std::max(0, -context_start);
    int down_pad = std::max(0, context_start + context_length - 1);
    PADDLE_ENFORCE_EQ(up_pad, 2, platform::errors::InvalidArgument(
                                     "Only support up_pad equal 2."));
    PADDLE_ENFORCE_EQ(down_pad, 2, platform::errors::InvalidArgument(
                                       "Only support down_pad equal 2."));

    auto lod_level_0 = in->lod()[0];
    int lod_size = lod_level_0.size();
    PADDLE_ENFORCE_LE(lod_size, 257, platform::errors::InvalidArgument(
                                         "Only support batch size <= 256."));

    std::vector<int> cpu_lodx(lod_size);
    for (int i = 0; i < lod_size; i++) {
      cpu_lodx[i] = lod_level_0[i];
    }
    xpu::VectorParam<int> lodx = {cpu_lodx.data(),
                                  static_cast<int>(cpu_lodx.size()), nullptr};

    auto xpu_context =
        context.template device_context<DeviceContext>().x_context();
    auto sequence_width = static_cast<int64_t>(in->dims()[1]);
    framework::DDim col_shape = {in->dims()[0],
                                 context_length * sequence_width};
    xpu::ctx_guard RAII_GUARD(xpu_context);
    int col_numel = col_shape[0] * col_shape[1];
    T* col_data = RAII_GUARD.alloc_l3_or_gm<T>(col_numel);
    PADDLE_ENFORCE_NOT_NULL(
        col_data, paddle::platform::errors::Fatal("XPU memory is not enough"));

    if (in_g || filter_g) {
      bool trans_a = false;
      bool trans_b = true;
      int m = out_g->dims()[0];
      int k = out_g->dims()[1];
      int n = filter->dims()[0];
      int k1 = filter->dims()[1];
      PADDLE_ENFORCE_EQ(k, k1,
                        platform::errors::InvalidArgument(
                            "The shape of FC in SequenceConvGradOp is invalid."
                            "The k of matrix A is %d, k1 of matrix B is %d."
                            "But expect k == k1",
                            k, k1));
      int lda = (!trans_a) ? k : m;
      int ldb = (!trans_b) ? n : k;
      int ldc = n;
      T alpha = static_cast<T>(1.0);
      T beta = static_cast<T>(0.0);
      const T* data_a = out_g->data<T>();
      const T* data_b = filter->data<T>();
      T* data_c = col_data;

      int r = xpu::fc_fusion<T, T, T, int32_t>(
          xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b,
          nullptr, nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
          xpu::Activation_t::LINEAR);
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
    }

    if (in_g) {
      PADDLE_ENFORCE_LT(sequence_width, 512,
                        platform::errors::InvalidArgument(
                            "Only support sequence_width < 512."));

      in_g->mutable_data<T>(context.GetPlace());
      in_g->set_lod(in->lod());

      int r = xpu::sequence_context_projection_grad<T, int>(
          xpu_context, in_g->data<T>(), col_data, nullptr, lodx, sequence_width,
          context_start, context_length, context_stride, {2, 2});
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection_grad");
    }

    if (filter_g) {
      filter_g->mutable_data<T>(context.GetPlace());

      int r = xpu::sequence_context_projection<T, int>(
          xpu_context, in->data<T>(), col_data, nullptr, lodx, sequence_width,
          context_start, context_length, context_stride, {2, 2});
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "sequence_context_projection");

      bool trans_a = true;
      bool trans_b = false;
      int k = col_shape[0];
      int m = col_shape[1];
      int k1 = out_g->dims()[0];
      int n = out_g->dims()[1];
      PADDLE_ENFORCE_EQ(k, k1,
                        platform::errors::InvalidArgument(
                            "The shape of FC in SequenceConvGradOp is invalid."
                            "The k of matrix A is %d, k1 of matrix B is %d."
                            "But expect k == k1",
                            k, k1));
      int lda = (!trans_a) ? k : m;
      int ldb = (!trans_b) ? n : k;
      int ldc = n;
      T alpha = static_cast<T>(1.0);
      T beta = static_cast<T>(0.0);
      const T* data_a = col_data;
      const T* data_b = out_g->data<T>();
      T* data_c = filter_g->data<T>();

      r = xpu::fc_fusion<T, T, T, int32_t>(
          xpu_context, data_a, data_b, data_c, m, n, k, trans_a, trans_b,
          nullptr, nullptr, nullptr, lda, ldb, ldc, alpha, beta, nullptr,
          xpu::Activation_t::LINEAR);
      PADDLE_ENFORCE_XDNN_SUCCESS(r, "fc_fusion");
      if (xpu_context->xpu_stream != nullptr) {
        xpu_wait(xpu_context->xpu_stream);
      }
    }
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP_XPU_KERNEL(
    sequence_conv,
    ops::SequenceConvXPUKernel<paddle::platform::XPUDeviceContext, float>);

REGISTER_OP_XPU_KERNEL(
    sequence_conv_grad,
    ops::SequenceConvGradXPUKernel<paddle::platform::XPUDeviceContext, float>);

#endif
<|endoftext|>