<commit_before>/**
 *  Copyright (c) 2014-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under both the Apache 2.0 license (found in the
 *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
 *  in the COPYING file in the root directory of this source tree).
 *  You may select, at your option, one of the above-listed licenses.
 */

#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
#include <net/if.h>

#include <boost/algorithm/string/trim.hpp>

#include <osquery/core.h>
#include <osquery/logger.h>
#include <osquery/tables.h>

#include "osquery/tables/networking/utils.h"

namespace osquery {
namespace tables {

#define MAX_NETLINK_SIZE 8192
#define MAX_NETLINK_ATTEMPTS 8

constexpr auto kDefaultIpv6Route = "::";
constexpr auto kDefaultIpv4Route = "0.0.0.0";

std::string getNetlinkIP(int family, const char* buffer) {
  char dst[INET6_ADDRSTRLEN] = {0};

  if (inet_ntop(family, buffer, dst, INET6_ADDRSTRLEN) == nullptr) {
    LOG(ERROR) << "Unsupported address family: " << family;
    return "";
  }
  std::string address(dst);
  boost::trim(address);

  return address;
}

std::string getDefaultRouteIP(int family) {
  switch (family) {
  case AF_INET:
    return kDefaultIpv4Route;
  case AF_INET6:
    return kDefaultIpv6Route;
  default:
    LOG(ERROR) << "Unsupported address family: " << family;
    return "";
  }
}

Status readNetlink(int socket_fd, int seq, char* output, size_t* size) {
  struct nlmsghdr* nl_hdr = nullptr;

  size_t message_size = 0;
  do {
    size_t latency = 0;
    size_t total_bytes = 0;
    ssize_t bytes = 0;
    while (bytes == 0) {
      bytes = recv(socket_fd, output, MAX_NETLINK_SIZE - message_size, 0);
      if (bytes < 0) {
        // Unrecoverable NETLINK error, bail.
        return Status(1, "Could not read from NETLINK");
      }

      // Bytes were returned by we might need to read more.
      total_bytes += bytes;
      if (latency >= MAX_NETLINK_ATTEMPTS) {
        // Too many attempts to read bytes have occurred.
        // Prevent the NETLINK socket from handing and bail.
        return Status(1, "Netlink timeout");
      } else if (bytes == 0) {
        if (total_bytes > 0) {
          // Bytes were read, but now no more are available, attempt to parse
          // the received NETLINK message.
          break;
        }
        ::usleep(20);
        latency += 1;
      }
    }

    // Assure valid header response, and not an error type.
    nl_hdr = (struct nlmsghdr*)output;
    if (NLMSG_OK(nl_hdr, bytes) == 0 || nl_hdr->nlmsg_type == NLMSG_ERROR) {
      return Status(1, "Read invalid NETLINK message");
    }

    if (nl_hdr->nlmsg_type == NLMSG_DONE) {
      break;
    }

    // Move the buffer pointer
    output += bytes;
    message_size += bytes;
    if ((nl_hdr->nlmsg_flags & NLM_F_MULTI) == 0) {
      break;
    }
  } while (static_cast<pid_t>(nl_hdr->nlmsg_seq) != seq ||
           static_cast<pid_t>(nl_hdr->nlmsg_pid) != getpid());

  *size = message_size;
  return Status(0, "OK");
}

void genNetlinkRoutes(const struct nlmsghdr* netlink_msg, QueryData& results) {
  std::string address;
  int mask = 0;
  char interface[IF_NAMESIZE] = {0};

  struct rtmsg* message = static_cast<struct rtmsg*>(NLMSG_DATA(netlink_msg));
  struct rtattr* attr = static_cast<struct rtattr*>(RTM_RTA(message));
  uint32_t attr_size = RTM_PAYLOAD(netlink_msg);

  Row r;

  // Iterate over each route in the netlink message
  bool has_destination = false;
  r["metric"] = "0";
  r["hopcount"] = INTEGER(0);
  while (RTA_OK(attr, attr_size)) {
    switch (attr->rta_type) {
    case RTA_OIF:
      if_indextoname(*(int*)RTA_DATA(attr), interface);
      r["interface"] = std::string(interface);
      break;
    case RTA_GATEWAY:
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["gateway"] = address;
      break;
    case RTA_PREFSRC:
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["source"] = address;
      break;
    case RTA_DST:
      if (message->rtm_dst_len != 32 && message->rtm_dst_len != 128) {
        mask = (int)message->rtm_dst_len;
      }
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["destination"] = address;
      has_destination = true;
      break;
    case RTA_PRIORITY:
      r["metric"] = INTEGER(*(int*)RTA_DATA(attr));
      break;
    case RTA_METRICS:
      struct rtattr* xattr = static_cast<struct rtattr*> RTA_DATA(attr);
      auto xattr_size = RTA_PAYLOAD(attr);
      while (RTA_OK(xattr, xattr_size)) {
        switch (xattr->rta_type) {
        case RTAX_HOPLIMIT:
          r["hopcount"] = INTEGER(*reinterpret_cast<int*>(RTA_DATA(xattr)));
          break;
        }
        xattr = RTA_NEXT(xattr, xattr_size);
      }
      break;
    }
    attr = RTA_NEXT(attr, attr_size);
  }

  if (!has_destination) {
    r["destination"] = getDefaultRouteIP(message->rtm_family);
    if (message->rtm_dst_len) {
      mask = (int)message->rtm_dst_len;
    }
  }

  // Route type determination
  if (message->rtm_type == RTN_UNICAST) {
    r["type"] = "gateway";
  } else if (message->rtm_type == RTN_LOCAL) {
    r["type"] = "local";
  } else if (message->rtm_type == RTN_BROADCAST) {
    r["type"] = "broadcast";
  } else if (message->rtm_type == RTN_ANYCAST) {
    r["type"] = "anycast";
  } else {
    r["type"] = "other";
  }

  r["flags"] = INTEGER(message->rtm_flags);

  // This is the cidr-formatted mask
  r["netmask"] = INTEGER(mask);

  // Fields not supported by Linux routes:
  r["mtu"] = "0";
  results.push_back(r);
}

QueryData genRoutes(QueryContext& context) {
  QueryData results;

  int socket_fd = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE);
  if (socket_fd < 0) {
    VLOG(1) << "Cannot open NETLINK socket";
    return {};
  }

  // Create netlink message header
  auto netlink_buffer = (void*)malloc(MAX_NETLINK_SIZE);
  if (netlink_buffer == nullptr) {
    close(socket_fd);
    return {};
  }

  memset(netlink_buffer, 0, MAX_NETLINK_SIZE);
  auto netlink_msg = (struct nlmsghdr*)netlink_buffer;
  netlink_msg->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
  netlink_msg->nlmsg_type = RTM_GETROUTE; // routes from kernel routing table
  netlink_msg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST | NLM_F_ATOMIC;
  netlink_msg->nlmsg_seq = 0;
  netlink_msg->nlmsg_pid = getpid();

  // Send the netlink request to the kernel
  if (send(socket_fd, netlink_msg, netlink_msg->nlmsg_len, 0) < 0) {
    TLOG << "Cannot write NETLINK request header to socket";
    close(socket_fd);
    free(netlink_buffer);
    return {};
  }

  // Wrap the read socket to support multi-netlink messages
  size_t size = 0;
  if (!readNetlink(socket_fd, 1, (char*)netlink_msg, &size).ok()) {
    TLOG << "Cannot read NETLINK response from socket";
    close(socket_fd);
    free(netlink_buffer);
    return {};
  }

  // Treat the netlink response as route information
  while (NLMSG_OK(netlink_msg, size)) {
    genNetlinkRoutes(netlink_msg, results);
    netlink_msg = NLMSG_NEXT(netlink_msg, size);
  }

  close(socket_fd);
  free(netlink_buffer);
  return results;
}
}
}
<commit_msg>route table: get the value of the MTU on Linux (#4981)<commit_after>/**
 *  Copyright (c) 2014-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under both the Apache 2.0 license (found in the
 *  LICENSE file in the root directory of this source tree) and the GPLv2 (found
 *  in the COPYING file in the root directory of this source tree).
 *  You may select, at your option, one of the above-listed licenses.
 */

#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
#include <net/if.h>

#include <boost/algorithm/string/trim.hpp>

#include <osquery/core.h>
#include <osquery/logger.h>
#include <osquery/tables.h>

#include "osquery/tables/networking/utils.h"

namespace osquery {
namespace tables {

#define MAX_NETLINK_SIZE 8192
#define MAX_NETLINK_ATTEMPTS 8

constexpr auto kDefaultIpv6Route = "::";
constexpr auto kDefaultIpv4Route = "0.0.0.0";

std::string getNetlinkIP(int family, const char* buffer) {
  char dst[INET6_ADDRSTRLEN] = {0};

  if (inet_ntop(family, buffer, dst, INET6_ADDRSTRLEN) == nullptr) {
    LOG(ERROR) << "Unsupported address family: " << family;
    return "";
  }
  std::string address(dst);
  boost::trim(address);

  return address;
}

std::string getDefaultRouteIP(int family) {
  switch (family) {
  case AF_INET:
    return kDefaultIpv4Route;
  case AF_INET6:
    return kDefaultIpv6Route;
  default:
    LOG(ERROR) << "Unsupported address family: " << family;
    return "";
  }
}

Status readNetlink(int socket_fd, int seq, char* output, size_t* size) {
  struct nlmsghdr* nl_hdr = nullptr;

  size_t message_size = 0;
  do {
    size_t latency = 0;
    size_t total_bytes = 0;
    ssize_t bytes = 0;
    while (bytes == 0) {
      bytes = recv(socket_fd, output, MAX_NETLINK_SIZE - message_size, 0);
      if (bytes < 0) {
        // Unrecoverable NETLINK error, bail.
        return Status(1, "Could not read from NETLINK");
      }

      // Bytes were returned by we might need to read more.
      total_bytes += bytes;
      if (latency >= MAX_NETLINK_ATTEMPTS) {
        // Too many attempts to read bytes have occurred.
        // Prevent the NETLINK socket from handing and bail.
        return Status(1, "Netlink timeout");
      } else if (bytes == 0) {
        if (total_bytes > 0) {
          // Bytes were read, but now no more are available, attempt to parse
          // the received NETLINK message.
          break;
        }
        ::usleep(20);
        latency += 1;
      }
    }

    // Assure valid header response, and not an error type.
    nl_hdr = (struct nlmsghdr*)output;
    if (NLMSG_OK(nl_hdr, bytes) == 0 || nl_hdr->nlmsg_type == NLMSG_ERROR) {
      return Status(1, "Read invalid NETLINK message");
    }

    if (nl_hdr->nlmsg_type == NLMSG_DONE) {
      break;
    }

    // Move the buffer pointer
    output += bytes;
    message_size += bytes;
    if ((nl_hdr->nlmsg_flags & NLM_F_MULTI) == 0) {
      break;
    }
  } while (static_cast<pid_t>(nl_hdr->nlmsg_seq) != seq ||
           static_cast<pid_t>(nl_hdr->nlmsg_pid) != getpid());

  *size = message_size;
  return Status(0, "OK");
}

void genNetlinkRoutes(const struct nlmsghdr* netlink_msg, QueryData& results) {
  std::string address;
  int mask = 0;
  char interface[IF_NAMESIZE] = {0};

  struct rtmsg* message = static_cast<struct rtmsg*>(NLMSG_DATA(netlink_msg));
  struct rtattr* attr = static_cast<struct rtattr*>(RTM_RTA(message));
  uint32_t attr_size = RTM_PAYLOAD(netlink_msg);

  Row r;

  // Iterate over each route in the netlink message
  bool has_destination = false;
  r["metric"] = "0";
  r["hopcount"] = INTEGER(0);
  r["mtu"] = INTEGER(0);
  while (RTA_OK(attr, attr_size)) {
    switch (attr->rta_type) {
    case RTA_OIF:
      if_indextoname(*(int*)RTA_DATA(attr), interface);
      r["interface"] = std::string(interface);
      break;
    case RTA_GATEWAY:
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["gateway"] = address;
      break;
    case RTA_PREFSRC:
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["source"] = address;
      break;
    case RTA_DST:
      if (message->rtm_dst_len != 32 && message->rtm_dst_len != 128) {
        mask = (int)message->rtm_dst_len;
      }
      address = getNetlinkIP(message->rtm_family, (char*)RTA_DATA(attr));
      r["destination"] = address;
      has_destination = true;
      break;
    case RTA_PRIORITY:
      r["metric"] = INTEGER(*(int*)RTA_DATA(attr));
      break;
    case RTA_METRICS:
      struct rtattr* xattr = static_cast<struct rtattr*> RTA_DATA(attr);
      auto xattr_size = RTA_PAYLOAD(attr);
      while (RTA_OK(xattr, xattr_size)) {
        switch (xattr->rta_type) {
        case RTAX_MTU:
          r["mtu"] = INTEGER(*reinterpret_cast<int*>(RTA_DATA(xattr)));
          break;
        case RTAX_HOPLIMIT:
          r["hopcount"] = INTEGER(*reinterpret_cast<int*>(RTA_DATA(xattr)));
          break;
        }
        xattr = RTA_NEXT(xattr, xattr_size);
      }
      break;
    }
    attr = RTA_NEXT(attr, attr_size);
  }

  if (!has_destination) {
    r["destination"] = getDefaultRouteIP(message->rtm_family);
    if (message->rtm_dst_len) {
      mask = (int)message->rtm_dst_len;
    }
  }

  // Route type determination
  if (message->rtm_type == RTN_UNICAST) {
    r["type"] = "gateway";
  } else if (message->rtm_type == RTN_LOCAL) {
    r["type"] = "local";
  } else if (message->rtm_type == RTN_BROADCAST) {
    r["type"] = "broadcast";
  } else if (message->rtm_type == RTN_ANYCAST) {
    r["type"] = "anycast";
  } else {
    r["type"] = "other";
  }

  r["flags"] = INTEGER(message->rtm_flags);

  // This is the cidr-formatted mask
  r["netmask"] = INTEGER(mask);

  results.push_back(r);
}

QueryData genRoutes(QueryContext& context) {
  QueryData results;

  int socket_fd = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE);
  if (socket_fd < 0) {
    VLOG(1) << "Cannot open NETLINK socket";
    return {};
  }

  // Create netlink message header
  auto netlink_buffer = (void*)malloc(MAX_NETLINK_SIZE);
  if (netlink_buffer == nullptr) {
    close(socket_fd);
    return {};
  }

  memset(netlink_buffer, 0, MAX_NETLINK_SIZE);
  auto netlink_msg = (struct nlmsghdr*)netlink_buffer;
  netlink_msg->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
  netlink_msg->nlmsg_type = RTM_GETROUTE; // routes from kernel routing table
  netlink_msg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST | NLM_F_ATOMIC;
  netlink_msg->nlmsg_seq = 0;
  netlink_msg->nlmsg_pid = getpid();

  // Send the netlink request to the kernel
  if (send(socket_fd, netlink_msg, netlink_msg->nlmsg_len, 0) < 0) {
    TLOG << "Cannot write NETLINK request header to socket";
    close(socket_fd);
    free(netlink_buffer);
    return {};
  }

  // Wrap the read socket to support multi-netlink messages
  size_t size = 0;
  if (!readNetlink(socket_fd, 1, (char*)netlink_msg, &size).ok()) {
    TLOG << "Cannot read NETLINK response from socket";
    close(socket_fd);
    free(netlink_buffer);
    return {};
  }

  // Treat the netlink response as route information
  while (NLMSG_OK(netlink_msg, size)) {
    genNetlinkRoutes(netlink_msg, results);
    netlink_msg = NLMSG_NEXT(netlink_msg, size);
  }

  close(socket_fd);
  free(netlink_buffer);
  return results;
}
}
}
<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/dom_ui/options/password_manager_handler.h"

#include "base/callback.h"
#include "base/stl_util-inl.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "base/values.h"
#include "chrome/browser/prefs/pref_service.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/common/pref_names.h"
#include "chrome/common/url_constants.h"
#include "grit/chromium_strings.h"
#include "grit/generated_resources.h"
#include "net/base/net_util.h"
#include "ui/base/l10n/l10n_util.h"

PasswordManagerHandler::PasswordManagerHandler()
    : ALLOW_THIS_IN_INITIALIZER_LIST(populater_(this)),
      ALLOW_THIS_IN_INITIALIZER_LIST(exception_populater_(this)) {
}

PasswordManagerHandler::~PasswordManagerHandler() {
}

void PasswordManagerHandler::GetLocalizedValues(
    DictionaryValue* localized_strings) {
  DCHECK(localized_strings);

  localized_strings->SetString("passwordsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_EXCEPTIONS_WINDOW_TITLE));
  localized_strings->SetString("savedPasswordsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_SHOW_PASSWORDS_TAB_TITLE));
  localized_strings->SetString("passwordExceptionsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_EXCEPTIONS_TAB_TITLE));
  localized_strings->SetString("passwordShowButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_SHOW_BUTTON));
  localized_strings->SetString("passwordHideButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_HIDE_BUTTON));
  localized_strings->SetString("passwordsSiteColumn",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_SITE_COLUMN));
  localized_strings->SetString("passwordsUsernameColumn",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_USERNAME_COLUMN));
  localized_strings->SetString("passwordsRemoveButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_REMOVE_BUTTON));
  localized_strings->SetString("passwordsRemoveAllButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_REMOVE_ALL_BUTTON));
  localized_strings->SetString("passwordsRemoveAllTitle",
      l10n_util::GetStringUTF16(
          IDS_PASSWORDS_PAGE_VIEW_CAPTION_DELETE_ALL_PASSWORDS));
  localized_strings->SetString("passwordsRemoveAllWarning",
      l10n_util::GetStringUTF16(
          IDS_PASSWORDS_PAGE_VIEW_TEXT_DELETE_ALL_PASSWORDS));
}

void PasswordManagerHandler::Initialize() {
  // We should not cache dom_ui_->GetProfile(). See crosbug.com/6304.
}

void PasswordManagerHandler::RegisterMessages() {
  DCHECK(dom_ui_);

  dom_ui_->RegisterMessageCallback("updatePasswordLists",
      NewCallback(this, &PasswordManagerHandler::UpdatePasswordLists));
  dom_ui_->RegisterMessageCallback("removeSavedPassword",
      NewCallback(this, &PasswordManagerHandler::RemoveSavedPassword));
  dom_ui_->RegisterMessageCallback("removePasswordException",
      NewCallback(this, &PasswordManagerHandler::RemovePasswordException));
  dom_ui_->RegisterMessageCallback("removeAllSavedPasswords",
      NewCallback(this, &PasswordManagerHandler::RemoveAllSavedPasswords));
  dom_ui_->RegisterMessageCallback("removeAllPasswordExceptions", NewCallback(
      this, &PasswordManagerHandler::RemoveAllPasswordExceptions));
}

PasswordStore* PasswordManagerHandler::GetPasswordStore() {
  return dom_ui_->GetProfile()->GetPasswordStore(Profile::EXPLICIT_ACCESS);
}

void PasswordManagerHandler::UpdatePasswordLists(const ListValue* args) {
  languages_ =
      dom_ui_->GetProfile()->GetPrefs()->GetString(prefs::kAcceptLanguages);
  populater_.Populate();
  exception_populater_.Populate();
}

void PasswordManagerHandler::RemoveSavedPassword(const ListValue* args) {
  std::string string_value = WideToUTF8(ExtractStringValue(args));
  int index;
  base::StringToInt(string_value, &index);

  GetPasswordStore()->RemoveLogin(*password_list_[index]);
  delete password_list_[index];
  password_list_.erase(password_list_.begin() + index);
  SetPasswordList();
}

void PasswordManagerHandler::RemovePasswordException(
    const ListValue* args) {
  std::string string_value = WideToUTF8(ExtractStringValue(args));
  int index;
  base::StringToInt(string_value, &index);

  GetPasswordStore()->RemoveLogin(*password_exception_list_[index]);
  delete password_exception_list_[index];
  password_exception_list_.erase(password_exception_list_.begin() + index);
  SetPasswordExceptionList();
}

void PasswordManagerHandler::RemoveAllSavedPasswords(
    const ListValue* args) {
  PasswordStore* store = GetPasswordStore();
  for (size_t i = 0; i < password_list_.size(); ++i)
    store->RemoveLogin(*password_list_[i]);
  STLDeleteElements(&password_list_);
  SetPasswordList();
}

void PasswordManagerHandler::RemoveAllPasswordExceptions(
    const ListValue* args) {
  PasswordStore* store = GetPasswordStore();
  for (size_t i = 0; i < password_exception_list_.size(); ++i)
    store->RemoveLogin(*password_exception_list_[i]);
  STLDeleteElements(&password_exception_list_);
  SetPasswordExceptionList();
}

void PasswordManagerHandler::SetPasswordList() {
  ListValue entries;
  for (size_t i = 0; i < password_list_.size(); ++i) {
    ListValue* entry = new ListValue();
    entry->Append(new StringValue(net::FormatUrl(password_list_[i]->origin,
                                                 languages_)));
    entry->Append(new StringValue(password_list_[i]->username_value));
    entry->Append(new StringValue(password_list_[i]->password_value));
    entries.Append(entry);
  }

  dom_ui_->CallJavascriptFunction(
      L"PasswordManager.setSavedPasswordsList", entries);
}

void PasswordManagerHandler::SetPasswordExceptionList() {
  ListValue entries;
  for (size_t i = 0; i < password_exception_list_.size(); ++i) {
    entries.Append(new StringValue(
        net::FormatUrl(password_exception_list_[i]->origin, languages_)));
  }

  dom_ui_->CallJavascriptFunction(
      L"PasswordManager.setPasswordExceptionsList", entries);
}

PasswordManagerHandler::ListPopulater::ListPopulater(
    PasswordManagerHandler* page) : page_(page),
                                    pending_login_query_(0) {
}

PasswordManagerHandler::ListPopulater::~ListPopulater() {
  page_->GetPasswordStore()->CancelLoginsQuery(pending_login_query_);
}

PasswordManagerHandler::PasswordListPopulater::PasswordListPopulater(
    PasswordManagerHandler* page) : ListPopulater(page) {
}

void PasswordManagerHandler::PasswordListPopulater::Populate() {
  DCHECK(!pending_login_query_);
  PasswordStore* store = page_->GetPasswordStore();
  if (store != NULL)
    pending_login_query_ = store->GetAutofillableLogins(this);
  else
    LOG(ERROR) << "No password store! Cannot display passwords.";
}

void PasswordManagerHandler::PasswordListPopulater::
    OnPasswordStoreRequestDone(int handle,
    const std::vector<webkit_glue::PasswordForm*>& result) {
  DCHECK_EQ(pending_login_query_, handle);
  pending_login_query_ = 0;
  page_->password_list_ = result;
  page_->SetPasswordList();
}

PasswordManagerHandler::PasswordExceptionListPopulater::
    PasswordExceptionListPopulater(PasswordManagerHandler* page)
        : ListPopulater(page) {
}

void PasswordManagerHandler::PasswordExceptionListPopulater::Populate() {
  DCHECK(!pending_login_query_);
  PasswordStore* store = page_->GetPasswordStore();
  if (store != NULL)
    pending_login_query_ = store->GetBlacklistLogins(this);
  else
    LOG(ERROR) << "No password store! Cannot display exceptions.";
}

void PasswordManagerHandler::PasswordExceptionListPopulater::
    OnPasswordStoreRequestDone(int handle,
    const std::vector<webkit_glue::PasswordForm*>& result) {
  DCHECK_EQ(pending_login_query_, handle);
  pending_login_query_ = 0;
  page_->password_exception_list_ = result;
  page_->SetPasswordExceptionList();
}
<commit_msg>DOMUI: Check for a NULL PasswordStore, as the PasswordStore may be gone by the time ListPopulater is destroyed.<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/dom_ui/options/password_manager_handler.h"

#include "base/callback.h"
#include "base/stl_util-inl.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "base/values.h"
#include "chrome/browser/prefs/pref_service.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/common/pref_names.h"
#include "chrome/common/url_constants.h"
#include "grit/chromium_strings.h"
#include "grit/generated_resources.h"
#include "net/base/net_util.h"
#include "ui/base/l10n/l10n_util.h"

PasswordManagerHandler::PasswordManagerHandler()
    : ALLOW_THIS_IN_INITIALIZER_LIST(populater_(this)),
      ALLOW_THIS_IN_INITIALIZER_LIST(exception_populater_(this)) {
}

PasswordManagerHandler::~PasswordManagerHandler() {
}

void PasswordManagerHandler::GetLocalizedValues(
    DictionaryValue* localized_strings) {
  DCHECK(localized_strings);

  localized_strings->SetString("passwordsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_EXCEPTIONS_WINDOW_TITLE));
  localized_strings->SetString("savedPasswordsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_SHOW_PASSWORDS_TAB_TITLE));
  localized_strings->SetString("passwordExceptionsTitle",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_EXCEPTIONS_TAB_TITLE));
  localized_strings->SetString("passwordShowButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_SHOW_BUTTON));
  localized_strings->SetString("passwordHideButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_HIDE_BUTTON));
  localized_strings->SetString("passwordsSiteColumn",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_SITE_COLUMN));
  localized_strings->SetString("passwordsUsernameColumn",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_USERNAME_COLUMN));
  localized_strings->SetString("passwordsRemoveButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_REMOVE_BUTTON));
  localized_strings->SetString("passwordsRemoveAllButton",
      l10n_util::GetStringUTF16(IDS_PASSWORDS_PAGE_VIEW_REMOVE_ALL_BUTTON));
  localized_strings->SetString("passwordsRemoveAllTitle",
      l10n_util::GetStringUTF16(
          IDS_PASSWORDS_PAGE_VIEW_CAPTION_DELETE_ALL_PASSWORDS));
  localized_strings->SetString("passwordsRemoveAllWarning",
      l10n_util::GetStringUTF16(
          IDS_PASSWORDS_PAGE_VIEW_TEXT_DELETE_ALL_PASSWORDS));
}

void PasswordManagerHandler::Initialize() {
  // We should not cache dom_ui_->GetProfile(). See crosbug.com/6304.
}

void PasswordManagerHandler::RegisterMessages() {
  DCHECK(dom_ui_);

  dom_ui_->RegisterMessageCallback("updatePasswordLists",
      NewCallback(this, &PasswordManagerHandler::UpdatePasswordLists));
  dom_ui_->RegisterMessageCallback("removeSavedPassword",
      NewCallback(this, &PasswordManagerHandler::RemoveSavedPassword));
  dom_ui_->RegisterMessageCallback("removePasswordException",
      NewCallback(this, &PasswordManagerHandler::RemovePasswordException));
  dom_ui_->RegisterMessageCallback("removeAllSavedPasswords",
      NewCallback(this, &PasswordManagerHandler::RemoveAllSavedPasswords));
  dom_ui_->RegisterMessageCallback("removeAllPasswordExceptions", NewCallback(
      this, &PasswordManagerHandler::RemoveAllPasswordExceptions));
}

PasswordStore* PasswordManagerHandler::GetPasswordStore() {
  return dom_ui_->GetProfile()->GetPasswordStore(Profile::EXPLICIT_ACCESS);
}

void PasswordManagerHandler::UpdatePasswordLists(const ListValue* args) {
  languages_ =
      dom_ui_->GetProfile()->GetPrefs()->GetString(prefs::kAcceptLanguages);
  populater_.Populate();
  exception_populater_.Populate();
}

void PasswordManagerHandler::RemoveSavedPassword(const ListValue* args) {
  std::string string_value = WideToUTF8(ExtractStringValue(args));
  int index;
  base::StringToInt(string_value, &index);

  GetPasswordStore()->RemoveLogin(*password_list_[index]);
  delete password_list_[index];
  password_list_.erase(password_list_.begin() + index);
  SetPasswordList();
}

void PasswordManagerHandler::RemovePasswordException(
    const ListValue* args) {
  std::string string_value = WideToUTF8(ExtractStringValue(args));
  int index;
  base::StringToInt(string_value, &index);

  GetPasswordStore()->RemoveLogin(*password_exception_list_[index]);
  delete password_exception_list_[index];
  password_exception_list_.erase(password_exception_list_.begin() + index);
  SetPasswordExceptionList();
}

void PasswordManagerHandler::RemoveAllSavedPasswords(
    const ListValue* args) {
  PasswordStore* store = GetPasswordStore();
  for (size_t i = 0; i < password_list_.size(); ++i)
    store->RemoveLogin(*password_list_[i]);
  STLDeleteElements(&password_list_);
  SetPasswordList();
}

void PasswordManagerHandler::RemoveAllPasswordExceptions(
    const ListValue* args) {
  PasswordStore* store = GetPasswordStore();
  for (size_t i = 0; i < password_exception_list_.size(); ++i)
    store->RemoveLogin(*password_exception_list_[i]);
  STLDeleteElements(&password_exception_list_);
  SetPasswordExceptionList();
}

void PasswordManagerHandler::SetPasswordList() {
  ListValue entries;
  for (size_t i = 0; i < password_list_.size(); ++i) {
    ListValue* entry = new ListValue();
    entry->Append(new StringValue(net::FormatUrl(password_list_[i]->origin,
                                                 languages_)));
    entry->Append(new StringValue(password_list_[i]->username_value));
    entry->Append(new StringValue(password_list_[i]->password_value));
    entries.Append(entry);
  }

  dom_ui_->CallJavascriptFunction(
      L"PasswordManager.setSavedPasswordsList", entries);
}

void PasswordManagerHandler::SetPasswordExceptionList() {
  ListValue entries;
  for (size_t i = 0; i < password_exception_list_.size(); ++i) {
    entries.Append(new StringValue(
        net::FormatUrl(password_exception_list_[i]->origin, languages_)));
  }

  dom_ui_->CallJavascriptFunction(
      L"PasswordManager.setPasswordExceptionsList", entries);
}

PasswordManagerHandler::ListPopulater::ListPopulater(
    PasswordManagerHandler* page) : page_(page),
                                    pending_login_query_(0) {
}

PasswordManagerHandler::ListPopulater::~ListPopulater() {
  PasswordStore* store = page_->GetPasswordStore();
  if (store)
    store->CancelLoginsQuery(pending_login_query_);
}

PasswordManagerHandler::PasswordListPopulater::PasswordListPopulater(
    PasswordManagerHandler* page) : ListPopulater(page) {
}

void PasswordManagerHandler::PasswordListPopulater::Populate() {
  DCHECK(!pending_login_query_);
  PasswordStore* store = page_->GetPasswordStore();
  if (store != NULL)
    pending_login_query_ = store->GetAutofillableLogins(this);
  else
    LOG(ERROR) << "No password store! Cannot display passwords.";
}

void PasswordManagerHandler::PasswordListPopulater::
    OnPasswordStoreRequestDone(int handle,
    const std::vector<webkit_glue::PasswordForm*>& result) {
  DCHECK_EQ(pending_login_query_, handle);
  pending_login_query_ = 0;
  page_->password_list_ = result;
  page_->SetPasswordList();
}

PasswordManagerHandler::PasswordExceptionListPopulater::
    PasswordExceptionListPopulater(PasswordManagerHandler* page)
        : ListPopulater(page) {
}

void PasswordManagerHandler::PasswordExceptionListPopulater::Populate() {
  DCHECK(!pending_login_query_);
  PasswordStore* store = page_->GetPasswordStore();
  if (store != NULL)
    pending_login_query_ = store->GetBlacklistLogins(this);
  else
    LOG(ERROR) << "No password store! Cannot display exceptions.";
}

void PasswordManagerHandler::PasswordExceptionListPopulater::
    OnPasswordStoreRequestDone(int handle,
    const std::vector<webkit_glue::PasswordForm*>& result) {
  DCHECK_EQ(pending_login_query_, handle);
  pending_login_query_ = 0;
  page_->password_exception_list_ = result;
  page_->SetPasswordExceptionList();
}
<|endoftext|>


<commit_before>///
/// @file  popcnt.hpp
/// @brief Functions to count the number of 1 bits inside
///        an array or a 64-bit word.
///
/// Copyright (C) 2020 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#ifndef POPCNT_HPP
#define POPCNT_HPP

#include <stdint.h>

#if defined(__has_include)
  #define HAS_INCLUDE(header) __has_include(header)
#else
  // If the __has_include() macro does not exist
  // we assume that the header file exists.
  #define HAS_INCLUDE(header) 1
#endif

#if defined(ENABLE_POPCNT)

#if !defined(__has_builtin)
  #define __has_builtin(x) 0
#endif

// GCC & Clang
#if defined(__GNUC__) || \
    __has_builtin(__builtin_popcountll)

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return __builtin_popcountll(x);
}

} // namespace

#elif defined(_MSC_VER) && \
      defined(_WIN64) && \
      HAS_INCLUDE(<nmmintrin.h>)

#include <nmmintrin.h>

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return _mm_popcnt_u64(x);
}

} // namespace

#elif defined(_MSC_VER) && \
      defined(_WIN32) && \
      HAS_INCLUDE(<nmmintrin.h>)

#include <nmmintrin.h>

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return _mm_popcnt_u32((uint32_t) x) +
         _mm_popcnt_u32((uint32_t)(x >> 32));
}

} // namespace

#else

// Since hardware popcount support is very important for
// performance in primecount we want the compilation to fail in
// case the compiler does not select any of the implementations
// above (which use compiler intrinsics to enable hardware
// popcount support). This way we can be relatively sure that
// when primecount has been compiled successfully, primecount
// will have hardware popcount support.
#error "No fast popcount function implemented for this compiler!"

#endif
#endif

namespace {

#if !defined(ENABLE_POPCNT)

/// This uses fewer arithmetic operations than any other known
/// implementation on machines with fast multiplication.
/// It uses 12 arithmetic operations, one of which is a multiply.
/// https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation
///
inline uint64_t popcnt64(uint64_t x)
{
  uint64_t m1 = 0x5555555555555555ull;
  uint64_t m2 = 0x3333333333333333ull;
  uint64_t m4 = 0x0F0F0F0F0F0F0F0Full;
  uint64_t h01 = 0x0101010101010101ull;

  x -= (x >> 1) & m1;
  x = (x & m2) + ((x >> 2) & m2);
  x = (x + (x >> 4)) & m4;

  return (x * h01) >> 56;
}

#endif

} // namespace

#endif // POPCNT_HPP
<commit_msg>Fallback to std::popcount<commit_after>///
/// @file  popcnt.hpp
/// @brief Functions to count the number of 1 bits inside
///        an array or a 64-bit word.
///
/// Copyright (C) 2020 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#ifndef POPCNT_HPP
#define POPCNT_HPP

#include <stdint.h>

#if defined(__has_include)
  #define HAS_INCLUDE(header) __has_include(header)
#else
  // If the __has_include() macro does not exist
  // we assume that the header file exists.
  #define HAS_INCLUDE(header) 1
#endif

#if defined(ENABLE_POPCNT)

#if !defined(__has_builtin)
  #define __has_builtin(x) 0
#endif

// GCC & Clang
#if defined(__GNUC__) || \
    __has_builtin(__builtin_popcountll)

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return __builtin_popcountll(x);
}

} // namespace

#elif defined(_MSC_VER) && \
      defined(_WIN64) && \
      HAS_INCLUDE(<nmmintrin.h>)

#include <nmmintrin.h>

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return _mm_popcnt_u64(x);
}

} // namespace

#elif defined(_MSC_VER) && \
      defined(_WIN32) && \
      HAS_INCLUDE(<nmmintrin.h>)

#include <nmmintrin.h>

namespace {

inline uint64_t popcnt64(uint64_t x)
{
  return _mm_popcnt_u32((uint32_t) x) +
         _mm_popcnt_u32((uint32_t)(x >> 32));
}

} // namespace

#else

#include <bit>

namespace {

/// Ideally we would like to std::popcount instead of the
/// intrinsics above. However std::popcount is from C++20 and
/// not yet widely supported. Also the assembly generated by
/// std::popcount is not as good as __builtin_popcountll on
/// x64 with the Clang compiler. This will likely be fixed in
/// a few years.
///
inline uint64_t popcnt64(uint64_t x)
{
  return std::popcount(x);
}

} // namespace

#endif
#endif

namespace {

#if !defined(ENABLE_POPCNT)

/// This uses fewer arithmetic operations than any other known
/// implementation on machines with fast multiplication.
/// It uses 12 arithmetic operations, one of which is a multiply.
/// https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation
///
inline uint64_t popcnt64(uint64_t x)
{
  uint64_t m1 = 0x5555555555555555ull;
  uint64_t m2 = 0x3333333333333333ull;
  uint64_t m4 = 0x0F0F0F0F0F0F0F0Full;
  uint64_t h01 = 0x0101010101010101ull;

  x -= (x >> 1) & m1;
  x = (x & m2) + ((x >> 2) & m2);
  x = (x + (x >> 4)) & m4;

  return (x * h01) >> 56;
}

#endif

} // namespace

#endif // POPCNT_HPP
<|endoftext|>


<commit_before>///
/// @file   ptypes.hpp
/// @brief  Convenience functions and typedefs for the non standard
///         __int128_t integer type.
///
/// Copyright (C) 2014 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#ifndef PTYPES_HPP
#define PTYPES_HPP

#if defined(HAVE_INT128_T)

#include <stdint.h>

namespace primecount {

typedef int128_t maxint_t;

}

#elif defined(HAVE___INT128_T)

// int128_t is a typedef for __int128_t
#define HAVE_INT128_T

#include <ostream>
#include <string>

namespace primecount {

typedef __int128_t maxint_t;
typedef __int128_t int128_t;

inline std::ostream& operator<<(std::ostream& stream, int128_t n)
{
  std::string str;
  while (n > 0)
  {
    str += '0' + n % 10;
    n /= 10;
  }
  stream << std::string(str.rbegin(), str.rend());
  return stream;
}

} // namespace

#else /* int128_t not supported */

namespace primecount {

typedef int64_t maxint_t;

}

#endif

#endif /* PTYPES_HPP */
<commit_msg>Fix int64_t undefined type<commit_after>///
/// @file   ptypes.hpp
/// @brief  Convenience functions and typedefs for the non standard
///         __int128_t integer type.
///
/// Copyright (C) 2014 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#ifndef PTYPES_HPP
#define PTYPES_HPP

#include <stdint.h>

#if defined(HAVE_INT128_T)

namespace primecount {

typedef int128_t maxint_t;

}

#elif defined(HAVE___INT128_T)

// int128_t is a typedef for __int128_t
#define HAVE_INT128_T

#include <ostream>
#include <string>

namespace primecount {

typedef __int128_t maxint_t;
typedef __int128_t int128_t;

inline std::ostream& operator<<(std::ostream& stream, int128_t n)
{
  std::string str;
  while (n > 0)
  {
    str += '0' + n % 10;
    n /= 10;
  }
  stream << std::string(str.rbegin(), str.rend());
  return stream;
}

} // namespace

#else /* int128_t not supported */

namespace primecount {

typedef int64_t maxint_t;

}

#endif

#endif /* PTYPES_HPP */
<|endoftext|>


<commit_before>/******************************************************************************
 * Copyright 2017 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include <memory>
#include <string>
#include <thread>

#include "cyber/cyber.h"

#include "modules/drivers/velodyne/parser/velodyne_convert_component.h"

namespace apollo {
namespace drivers {
namespace velodyne {

bool VelodyneConvertComponent::Init() {
  Config velodyne_config;
  if (!GetProtoConfig(&velodyne_config)) {
    AWARN << "Load config failed, config file" << config_file_path_;
    return false;
  }

  conv_.reset(new Convert());
  conv_->init(velodyne_config);
  writer_ =
      node_->CreateWriter<PointCloud>(velodyne_config.convert_channel_name());
  point_cloud_pool_.reset(new CCObjectPool<PointCloud>(pool_size_));
  point_cloud_pool_->ConstructAll();
  for (int i = 0; i < pool_size_; i++) {
    auto point_cloud = point_cloud_pool_->GetObject();
    if (point_cloud == nullptr) {
      AERROR << "fail to getobject, i: " << i;
      return false;
    }
    point_cloud->mutable_point()->Reserve(140000);
  }
  AINFO << "Point cloud comp convert init success";
  return true;
}

bool VelodyneConvertComponent::Proc(
    const std::shared_ptr<VelodyneScan>& scan_msg) {
  std::shared_ptr<PointCloud> point_cloud_out = point_cloud_pool_->GetObject();
  if (point_cloud_out == nullptr) {
    AWARN << "poin cloud pool return nullptr, will be create new.";
    point_cloud_out = std::make_shared<PointCloud>();
    point_cloud_out->mutable_point()->Reserve(140000);
  }
  if (point_cloud_out == nullptr) {
    AWARN << "point cloud out is nullptr";
    return false;
  }
  AINFO << "Clear before bytes:" << point_cloud_out->ByteSize();
  point_cloud_out->Clear();
  AINFO << "Clear after bytes:" << point_cloud_out->ByteSize();
  AINFO << "SpaceUsedExcludingSelf: "
        << point_cloud_out->mutable_point()->SpaceUsedExcludingSelf()
        << ", ClearedCount:"
        << point_cloud_out->mutable_point()->ClearedCount();
  conv_->ConvertPacketsToPointcloud(scan_msg, point_cloud_out);

  if (point_cloud_out == nullptr || point_cloud_out->point_size() == 0) {
    AWARN << "point_cloud_out convert is empty.";
    return false;
  }
  writer_->Write(point_cloud_out);
  return true;
}

}  // namespace velodyne
}  // namespace drivers
}  // namespace apollo
<commit_msg>Drivers: clear velodyne debug log<commit_after>/******************************************************************************
 * Copyright 2017 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include <memory>
#include <string>
#include <thread>

#include "cyber/cyber.h"

#include "modules/drivers/velodyne/parser/velodyne_convert_component.h"

namespace apollo {
namespace drivers {
namespace velodyne {

bool VelodyneConvertComponent::Init() {
  Config velodyne_config;
  if (!GetProtoConfig(&velodyne_config)) {
    AWARN << "Load config failed, config file" << config_file_path_;
    return false;
  }

  conv_.reset(new Convert());
  conv_->init(velodyne_config);
  writer_ =
      node_->CreateWriter<PointCloud>(velodyne_config.convert_channel_name());
  point_cloud_pool_.reset(new CCObjectPool<PointCloud>(pool_size_));
  point_cloud_pool_->ConstructAll();
  for (int i = 0; i < pool_size_; i++) {
    auto point_cloud = point_cloud_pool_->GetObject();
    if (point_cloud == nullptr) {
      AERROR << "fail to getobject, i: " << i;
      return false;
    }
    point_cloud->mutable_point()->Reserve(140000);
  }
  AINFO << "Point cloud comp convert init success";
  return true;
}

bool VelodyneConvertComponent::Proc(
    const std::shared_ptr<VelodyneScan>& scan_msg) {
  std::shared_ptr<PointCloud> point_cloud_out = point_cloud_pool_->GetObject();
  if (point_cloud_out == nullptr) {
    AWARN << "poin cloud pool return nullptr, will be create new.";
    point_cloud_out = std::make_shared<PointCloud>();
    point_cloud_out->mutable_point()->Reserve(140000);
  }
  if (point_cloud_out == nullptr) {
    AWARN << "point cloud out is nullptr";
    return false;
  }
  point_cloud_out->Clear();
  conv_->ConvertPacketsToPointcloud(scan_msg, point_cloud_out);

  if (point_cloud_out == nullptr || point_cloud_out->point_size() == 0) {
    AWARN << "point_cloud_out convert is empty.";
    return false;
  }
  writer_->Write(point_cloud_out);
  return true;
}

}  // namespace velodyne
}  // namespace drivers
}  // namespace apollo
<|endoftext|>


<commit_before><commit_msg>Prediction: also SetNearbyObstacles for ego_vehicle<commit_after><|endoftext|>


<commit_before>#include <iostream>
#include <cstdio>
#include <algorithm>
#include <queue>
#include <map>
#include <set>
#include <stack>
#include <cstring>
#include <string>
#include <vector>
#include <iomanip>
#include <cmath>
#include <list>
#include <bitset>

using namespace std;

#define ll long long
#define lson l,mid,id<<1
#define rson mid+1,r,id<<1|1

typedef pair<int, int>pii;
typedef pair<ll, ll>pll;
typedef pair<double, double>pdd;

const double eps = 1e-6;
const ll  LINF = 0x3f3f3f3f3f3f3f3fLL;
const int INF = 0x3f3f3f3f;
const double FINF = 1e18;

#define x first
#define y second

#define REP(i,j,k) for(int i =(j);i<=(k);i++)
#define REPD(i,j,k) for(int i =(j);i>=(k);i--)
#define print(x) cout<<(x)<<endl;
#define IOS ios::sync_with_stdio(0);cin.tie(0);

void Init(){

	return ;
}

void Solve(int t){
	int n;
	scanf("%d",&n);

	int a=n*n;

	REPD(i,2051,3){

		int b=i*i;

		int k1=sqrt(a+b);
			if(k1*k1==a+b&& k1>0 && i>=k1){
				printf("%d %d\n",i,k1);
			}
		int k2=sqrt(a-b);
			if(k2*k2+b==a&& k2>0 && i>=k2){
				printf("%d %d\n",i,k2);
			}
		int k3=sqrt(b-a);
			if(k3*k3+a==b&& k3>0 && i>=k3){
				printf("%d %d\n",i,k3);
			}
	}
	if(t>=1) printf("\n");

	return ;
}

int main(){

	freopen("uoj1079.in","r",stdin);

	int t;
	scanf("%d",&t);
	while(t--)
	Init(),Solve(t);
	return 0;
}<commit_msg>update uoj1079<commit_after>#include <iostream>
#include <cstdio>
#include <algorithm>
#include <queue>
#include <map>
#include <set>
#include <stack>
#include <cstring>
#include <string>
#include <vector>
#include <iomanip>
#include <cmath>
#include <list>
#include <bitset>

using namespace std;

#define ll long long
#define lson l,mid,id<<1
#define rson mid+1,r,id<<1|1

typedef pair<int, int>pii;
typedef pair<ll, ll>pll;
typedef pair<double, double>pdd;

const double eps = 1e-6;
const ll  LINF = 0x3f3f3f3f3f3f3f3fLL;
const int INF = 0x3f3f3f3f;
const double FINF = 1e18;

#define x first
#define y second

#define REP(i,j,k) for(int i =(j);i<=(k);i++)
#define REPD(i,j,k) for(int i =(j);i>=(k);i--)
#define print(x) cout<<(x)<<endl;
#define IOS ios::sync_with_stdio(0);cin.tie(0);

void Init(){

	return ;
}

void Solve(int t){
	int n;
	scanf("%d",&n);

	int a=n*n;

	REPD(i,2051,3){

		int b=i*i;

		int k1=sqrt(a+b);
			if(k1*k1==a+b&& k1>0 && i>=k1){
				printf("%d %d\n",i,k1);
			}
		int k2=sqrt(a-b);
			if(k2*k2+b==a&& k2>0 && i>=k2){
				printf("%d %d\n",i,k2);
			}
		int k3=sqrt(b-a);
			if(k3*k3+a==b&& k3>0 && i>=k3){
				printf("%d %d\n",i,k3);
			}
	}
	if(t>=1) printf("1\n");

	return ;
}

int main(){

	freopen("uoj1079.in","r",stdin);

	int t;
	scanf("%d",&t);
	while(t--)
	Init(),Solve(t);
	return 0;
}<|endoftext|>


<commit_before>// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/sync/engine/post_commit_message_command.h"

#include <vector>

#include "chrome/browser/sync/engine/syncer_proto_util.h"
#include "chrome/browser/sync/engine/syncproto.h"
#include "chrome/browser/sync/sessions/sync_session.h"
#include "chrome/browser/sync/syncable/directory_manager.h"
#include "chrome/browser/sync/util/sync_types.h"

using std::vector;

namespace browser_sync {

PostCommitMessageCommand::PostCommitMessageCommand() {}
PostCommitMessageCommand::~PostCommitMessageCommand() {}

void PostCommitMessageCommand::ExecuteImpl(sessions::SyncSession* session) {
  if (session->status_controller()->commit_ids().empty())
    return;  // Nothing to commit.
  ClientToServerResponse response;
  syncable::ScopedDirLookup dir(session->context()->directory_manager(),
                                session->context()->account_name());
  if (!dir.good())
    return;
  sessions::StatusController* status = session->status_controller();
  if (!SyncerProtoUtil::PostClientToServerMessage(
      status->mutable_commit_message(), &response, session)) {
    // None of our changes got through, let's clear sync flags and wait for
    // another list update.
    status->increment_num_consecutive_problem_commits();
    status->increment_num_consecutive_errors();
    syncable::WriteTransaction trans(dir, syncable::SYNCER, __FILE__, __LINE__);
    // TODO(sync): why set this flag, it seems like a bad side-effect?
    const vector<syncable::Id>& commit_ids = status->commit_ids();
    for (size_t i = 0; i < commit_ids.size(); i++) {
      syncable::MutableEntry entry(&trans, syncable::GET_BY_ID, commit_ids[i]);
      entry.Put(syncable::SYNCING, false);
    }
    return;
  } else {
    status->set_items_committed(true);
  }
  status->mutable_commit_response()->CopyFrom(response);
}

}  // namespace browser_sync
<commit_msg>Remove a TODO in post_commit_command.cc.  Clarify what's going on.<commit_after>// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/sync/engine/post_commit_message_command.h"

#include <vector>

#include "chrome/browser/sync/engine/syncer_proto_util.h"
#include "chrome/browser/sync/engine/syncproto.h"
#include "chrome/browser/sync/sessions/sync_session.h"
#include "chrome/browser/sync/syncable/directory_manager.h"
#include "chrome/browser/sync/util/sync_types.h"

using std::vector;

namespace browser_sync {

PostCommitMessageCommand::PostCommitMessageCommand() {}
PostCommitMessageCommand::~PostCommitMessageCommand() {}

void PostCommitMessageCommand::ExecuteImpl(sessions::SyncSession* session) {
  if (session->status_controller()->commit_ids().empty())
    return;  // Nothing to commit.
  ClientToServerResponse response;
  syncable::ScopedDirLookup dir(session->context()->directory_manager(),
                                session->context()->account_name());
  if (!dir.good())
    return;
  sessions::StatusController* status = session->status_controller();
  if (!SyncerProtoUtil::PostClientToServerMessage(
      status->mutable_commit_message(), &response, session)) {
    // None of our changes got through.  Clear the SYNCING bit which was
    // set to true during BuildCommitCommand, and which may still be true.
    // Not to be confused with IS_UNSYNCED.  This bit is used to detect local
    // changes to items that happen during the server Commit operation.
    status->increment_num_consecutive_problem_commits();
    status->increment_num_consecutive_errors();
    syncable::WriteTransaction trans(dir, syncable::SYNCER, __FILE__, __LINE__);
    const vector<syncable::Id>& commit_ids = status->commit_ids();
    for (size_t i = 0; i < commit_ids.size(); i++) {
      syncable::MutableEntry entry(&trans, syncable::GET_BY_ID, commit_ids[i]);
      entry.Put(syncable::SYNCING, false);
    }
    return;
  } else {
    status->set_items_committed(true);
  }
  status->mutable_commit_response()->CopyFrom(response);
}

}  // namespace browser_sync
<|endoftext|>


<commit_before>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "riegeli/bytes/message_parse.h"

#include <stddef.h>

#include <limits>
#include <tuple>

#include "absl/base/optimization.h"
#include "absl/strings/str_cat.h"
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/message_lite.h"
#include "riegeli/base/base.h"
#include "riegeli/base/canonical_errors.h"
#include "riegeli/base/chain.h"
#include "riegeli/base/status.h"
#include "riegeli/bytes/chain_reader.h"
#include "riegeli/bytes/reader.h"

namespace riegeli {

namespace {

// ReaderInputStream adapts a Reader to a ZeroCopyInputStream.
class ReaderInputStream : public google::protobuf::io::ZeroCopyInputStream {
 public:
  explicit ReaderInputStream(Reader* src)
      : src_(RIEGELI_ASSERT_NOTNULL(src)), initial_pos_(src_->pos()) {}

  bool Next(const void** data, int* size) override;
  void BackUp(int length) override;
  bool Skip(int length) override;
  google::protobuf::int64 ByteCount() const override;

 private:
  Position relative_pos() const;

  Reader* src_;
  // Invariants:
  //   src_->pos() >= initial_pos_
  //   src_->pos() - initial_pos_ <=
  //   numeric_limits<google::protobuf::int64>::max()
  Position initial_pos_;
};

inline Position ReaderInputStream::relative_pos() const {
  RIEGELI_ASSERT_GE(src_->pos(), initial_pos_)
      << "Failed invariant of ReaderInputStream: "
         "current position smaller than initial position";
  const Position pos = src_->pos() - initial_pos_;
  RIEGELI_ASSERT_LE(
      pos, Position{std::numeric_limits<google::protobuf::int64>::max()})
      << "Failed invariant of ReaderInputStream: "
         "relative position overflow";
  return pos;
}

bool ReaderInputStream::Next(const void** data, int* size) {
  const Position pos = relative_pos();
  if (ABSL_PREDICT_FALSE(
          pos ==
          Position{std::numeric_limits<google::protobuf::int64>::max()})) {
    return false;
  }
  if (ABSL_PREDICT_FALSE(!src_->Pull())) return false;
  *data = src_->cursor();
  *size = IntCast<int>(UnsignedMin(
      src_->available(), size_t{std::numeric_limits<int>::max()},
      Position{std::numeric_limits<google::protobuf::int64>::max()} - pos));
  src_->set_cursor(src_->cursor() + *size);
  return true;
}

void ReaderInputStream::BackUp(int length) {
  RIEGELI_ASSERT_GE(length, 0)
      << "Failed precondition of ZeroCopyInputStream::BackUp(): "
         "negative length";
  RIEGELI_ASSERT_LE(IntCast<size_t>(length), src_->read_from_buffer())
      << "Failed precondition of ZeroCopyInputStream::BackUp(): "
         "length larger than the amount of buffered data";
  src_->set_cursor(src_->cursor() - length);
}

bool ReaderInputStream::Skip(int length) {
  RIEGELI_ASSERT_GE(length, 0)
      << "Failed precondition of ZeroCopyInputStream::Skip(): negative length";
  const Position max_length =
      Position{std::numeric_limits<google::protobuf::int64>::max()} -
      relative_pos();
  if (ABSL_PREDICT_FALSE(IntCast<size_t>(length) > max_length)) {
    src_->Skip(max_length);
    return false;
  }
  return src_->Skip(IntCast<size_t>(length));
}

google::protobuf::int64 ReaderInputStream::ByteCount() const {
  return IntCast<google::protobuf::int64>(relative_pos());
}

}  // namespace

namespace internal {

Status ParseFromReaderImpl(google::protobuf::MessageLite* dest, Reader* src) {
  {
    const Status status = ParsePartialFromReaderImpl(dest, src);
    if (ABSL_PREDICT_FALSE(!status.ok())) return status;
  }
  if (ABSL_PREDICT_FALSE(!dest->IsInitialized())) {
    return DataLossError(
        absl::StrCat("Failed to parse message of type ", dest->GetTypeName(),
                     " because it is missing required fields: ",
                     dest->InitializationErrorString()));
  }
  return OkStatus();
}

Status ParsePartialFromReaderImpl(google::protobuf::MessageLite* dest,
                                  Reader* src) {
  ReaderInputStream input_stream(src);
  if (ABSL_PREDICT_FALSE(
          !dest->ParsePartialFromZeroCopyStream(&input_stream))) {
    return DataLossError(
        absl::StrCat("Failed to parse message of type ", dest->GetTypeName()));
  }
  return OkStatus();
}

}  // namespace internal

Status ParseFromChain(google::protobuf::MessageLite* dest, const Chain& src) {
  return ParseFromReader<ChainReader<>>(dest, std::forward_as_tuple(&src));
}

Status ParsePartialFromChain(google::protobuf::MessageLite* dest,
                             const Chain& src) {
  return ParsePartialFromReader<ChainReader<>>(dest,
                                               std::forward_as_tuple(&src));
}

}  // namespace riegeli
<commit_msg>Speed up Parse{,Partial}From{Reader,Chain}() for small inputs: if the input is flat, use ParsePartialFromArray() instead of ParsePartialFromZeroCopyStream().<commit_after>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "riegeli/bytes/message_parse.h"

#include <stddef.h>

#include <limits>
#include <tuple>

#include "absl/base/optimization.h"
#include "absl/strings/str_cat.h"
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/message_lite.h"
#include "riegeli/base/base.h"
#include "riegeli/base/canonical_errors.h"
#include "riegeli/base/chain.h"
#include "riegeli/base/status.h"
#include "riegeli/bytes/chain_reader.h"
#include "riegeli/bytes/reader.h"

namespace riegeli {

namespace {

// ReaderInputStream adapts a Reader to a ZeroCopyInputStream.
class ReaderInputStream : public google::protobuf::io::ZeroCopyInputStream {
 public:
  explicit ReaderInputStream(Reader* src)
      : src_(RIEGELI_ASSERT_NOTNULL(src)), initial_pos_(src_->pos()) {}

  bool Next(const void** data, int* size) override;
  void BackUp(int length) override;
  bool Skip(int length) override;
  google::protobuf::int64 ByteCount() const override;

 private:
  Position relative_pos() const;

  Reader* src_;
  // Invariants:
  //   src_->pos() >= initial_pos_
  //   src_->pos() - initial_pos_ <=
  //   numeric_limits<google::protobuf::int64>::max()
  Position initial_pos_;
};

inline Position ReaderInputStream::relative_pos() const {
  RIEGELI_ASSERT_GE(src_->pos(), initial_pos_)
      << "Failed invariant of ReaderInputStream: "
         "current position smaller than initial position";
  const Position pos = src_->pos() - initial_pos_;
  RIEGELI_ASSERT_LE(
      pos, Position{std::numeric_limits<google::protobuf::int64>::max()})
      << "Failed invariant of ReaderInputStream: "
         "relative position overflow";
  return pos;
}

bool ReaderInputStream::Next(const void** data, int* size) {
  const Position pos = relative_pos();
  if (ABSL_PREDICT_FALSE(
          pos ==
          Position{std::numeric_limits<google::protobuf::int64>::max()})) {
    return false;
  }
  if (ABSL_PREDICT_FALSE(!src_->Pull())) return false;
  *data = src_->cursor();
  *size = IntCast<int>(UnsignedMin(
      src_->available(), size_t{std::numeric_limits<int>::max()},
      Position{std::numeric_limits<google::protobuf::int64>::max()} - pos));
  src_->set_cursor(src_->cursor() + *size);
  return true;
}

void ReaderInputStream::BackUp(int length) {
  RIEGELI_ASSERT_GE(length, 0)
      << "Failed precondition of ZeroCopyInputStream::BackUp(): "
         "negative length";
  RIEGELI_ASSERT_LE(IntCast<size_t>(length), src_->read_from_buffer())
      << "Failed precondition of ZeroCopyInputStream::BackUp(): "
         "length larger than the amount of buffered data";
  src_->set_cursor(src_->cursor() - length);
}

bool ReaderInputStream::Skip(int length) {
  RIEGELI_ASSERT_GE(length, 0)
      << "Failed precondition of ZeroCopyInputStream::Skip(): negative length";
  const Position max_length =
      Position{std::numeric_limits<google::protobuf::int64>::max()} -
      relative_pos();
  if (ABSL_PREDICT_FALSE(IntCast<size_t>(length) > max_length)) {
    src_->Skip(max_length);
    return false;
  }
  return src_->Skip(IntCast<size_t>(length));
}

google::protobuf::int64 ReaderInputStream::ByteCount() const {
  return IntCast<google::protobuf::int64>(relative_pos());
}

}  // namespace

namespace internal {

Status ParseFromReaderImpl(google::protobuf::MessageLite* dest, Reader* src) {
  {
    const Status status = ParsePartialFromReaderImpl(dest, src);
    if (ABSL_PREDICT_FALSE(!status.ok())) return status;
  }
  if (ABSL_PREDICT_FALSE(!dest->IsInitialized())) {
    return DataLossError(
        absl::StrCat("Failed to parse message of type ", dest->GetTypeName(),
                     " because it is missing required fields: ",
                     dest->InitializationErrorString()));
  }
  return OkStatus();
}

Status ParsePartialFromReaderImpl(google::protobuf::MessageLite* dest,
                                  Reader* src) {
  if (src->SupportsRandomAccess()) {
    Position size;
    if (ABSL_PREDICT_FALSE(!src->Size(&size))) return src->status();
    src->Pull();
    if (src->pos() + src->available() == size &&
        ABSL_PREDICT_TRUE(src->available() <=
                          size_t{std::numeric_limits<int>::max()})) {
      // The data are flat. ParsePartialFromArray() is faster than
      // ParsePartialFromZeroCopyStream().
      bool ok = dest->ParsePartialFromArray(src->cursor(),
                                            IntCast<int>(src->available()));
      src->set_cursor(src->cursor() + src->available());
      if (ABSL_PREDICT_FALSE(!ok)) {
        return DataLossError(absl::StrCat("Failed to parse message of type ",
                                          dest->GetTypeName()));
      }
      return OkStatus();
    }
  }
  ReaderInputStream input_stream(src);
  if (ABSL_PREDICT_FALSE(
          !dest->ParsePartialFromZeroCopyStream(&input_stream))) {
    if (ABSL_PREDICT_FALSE(!src->healthy())) return src->status();
    return DataLossError(
        absl::StrCat("Failed to parse message of type ", dest->GetTypeName()));
  }
  return OkStatus();
}

}  // namespace internal

Status ParseFromChain(google::protobuf::MessageLite* dest, const Chain& src) {
  return ParseFromReader<ChainReader<>>(dest, std::forward_as_tuple(&src));
}

Status ParsePartialFromChain(google::protobuf::MessageLite* dest,
                             const Chain& src) {
  return ParsePartialFromReader<ChainReader<>>(dest,
                                               std::forward_as_tuple(&src));
}

}  // namespace riegeli
<|endoftext|>


<commit_before>///////////////////////////////////////////////////////////////////////
// File:        fullyconnected.cpp
// Description: Simple feed-forward layer with various non-linearities.
// Author:      Ray Smith
// Created:     Wed Feb 26 14:49:15 PST 2014
//
// (C) Copyright 2014, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
///////////////////////////////////////////////////////////////////////

#include "fullyconnected.h"

#ifdef _OPENMP
#include <omp.h>
#endif
#include <stdio.h>
#include <stdlib.h>

#include "functions.h"
#include "networkscratch.h"

// Number of threads to use for parallel calculation of Forward and Backward.
const int kNumThreads = 4;

namespace tesseract {

FullyConnected::FullyConnected(const STRING& name, int ni, int no,
                               NetworkType type)
  : Network(type, name, ni, no), external_source_(NULL), int_mode_(false) {
}

FullyConnected::~FullyConnected() {
}

// Returns the shape output from the network given an input shape (which may
// be partially unknown ie zero).
StaticShape FullyConnected::OutputShape(const StaticShape& input_shape) const {
  LossType loss_type = LT_NONE;
  if (type_ == NT_SOFTMAX)
    loss_type = LT_CTC;
  else if (type_ == NT_SOFTMAX_NO_CTC)
    loss_type = LT_SOFTMAX;
  else if (type_ == NT_LOGISTIC)
    loss_type = LT_LOGISTIC;
  StaticShape result(input_shape);
  result.set_depth(no_);
  result.set_loss_type(loss_type);
  return result;
}

// Suspends/Enables training by setting the training_ flag.
void FullyConnected::SetEnableTraining(TrainingState state) {
  if (state == TS_RE_ENABLE) {
    // Enable only from temp disabled.
    if (training_ == TS_TEMP_DISABLE) training_ = TS_ENABLED;
  } else if (state == TS_TEMP_DISABLE) {
    // Temp disable only from enabled.
    if (training_ == TS_ENABLED) training_ = state;
  } else {
    if (state == TS_ENABLED && training_ != TS_ENABLED)
      weights_.InitBackward();
    training_ = state;
  }
}

// Sets up the network for training. Initializes weights using weights of
// scale `range` picked according to the random number generator `randomizer`.
int FullyConnected::InitWeights(float range, TRand* randomizer) {
  Network::SetRandomizer(randomizer);
  num_weights_ = weights_.InitWeightsFloat(no_, ni_ + 1, TestFlag(NF_ADA_GRAD),
                                           range, randomizer);
  return num_weights_;
}

// Converts a float network to an int network.
void FullyConnected::ConvertToInt() {
  weights_.ConvertToInt();
}

// Provides debug output on the weights.
void FullyConnected::DebugWeights() {
  weights_.Debug2D(name_.string());
}

// Writes to the given file. Returns false in case of error.
bool FullyConnected::Serialize(TFile* fp) const {
  if (!Network::Serialize(fp)) return false;
  if (!weights_.Serialize(IsTraining(), fp)) return false;
  return true;
}

// Reads from the given file. Returns false in case of error.
bool FullyConnected::DeSerialize(TFile* fp) {
  return weights_.DeSerialize(IsTraining(), fp);
}

// Runs forward propagation of activations on the input line.
// See NetworkCpp for a detailed discussion of the arguments.
void FullyConnected::Forward(bool debug, const NetworkIO& input,
                             const TransposedArray* input_transpose,
                             NetworkScratch* scratch, NetworkIO* output) {
  int width = input.Width();
  if (type_ == NT_SOFTMAX)
    output->ResizeFloat(input, no_);
  else
    output->Resize(input, no_);
  SetupForward(input, input_transpose);
  GenericVector<NetworkScratch::FloatVec> temp_lines;
  temp_lines.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  GenericVector<NetworkScratch::FloatVec> curr_input;
  curr_input.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < temp_lines.size(); ++i) {
    temp_lines[i].Init(no_, scratch);
    curr_input[i].Init(ni_, scratch);
  }
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = 0;
#endif
    double* temp_line = temp_lines[thread_id];
    const double* d_input = NULL;
    const inT8* i_input = NULL;
    if (input.int_mode()) {
      i_input = input.i(t);
    } else {
      input.ReadTimeStep(t, curr_input[thread_id]);
      d_input = curr_input[thread_id];
    }
    ForwardTimeStep(d_input, i_input, t, temp_line);
    output->WriteTimeStep(t, temp_line);
    if (IsTraining() && type_ != NT_SOFTMAX) {
      acts_.CopyTimeStepFrom(t, *output, t);
    }
  }
  // Zero all the elements that are in the padding around images that allows
  // multiple different-sized images to exist in a single array.
  // acts_ is only used if this is not a softmax op.
  if (IsTraining() && type_ != NT_SOFTMAX) {
    acts_.ZeroInvalidElements();
  }
  output->ZeroInvalidElements();
#if DEBUG_DETAIL > 0
  tprintf("F Output:%s\n", name_.string());
  output->Print(10);
#endif
  if (debug) DisplayForward(*output);
}

// Components of Forward so FullyConnected can be reused inside LSTM.
void FullyConnected::SetupForward(const NetworkIO& input,
                                  const TransposedArray* input_transpose) {
  // Softmax output is always float, so save the input type.
  int_mode_ = input.int_mode();
  if (IsTraining()) {
    acts_.Resize(input, no_);
    // Source_ is a transposed copy of input. It isn't needed if provided.
    external_source_ = input_transpose;
    if (external_source_ == NULL) source_t_.ResizeNoInit(ni_, input.Width());
  }
}

void FullyConnected::ForwardTimeStep(const double* d_input, const inT8* i_input,
                                     int t, double* output_line) {
  // input is copied to source_ line-by-line for cache coherency.
  if (IsTraining() && external_source_ == NULL && d_input != NULL)
    source_t_.WriteStrided(t, d_input);
  if (d_input != NULL)
    weights_.MatrixDotVector(d_input, output_line);
  else
    weights_.MatrixDotVector(i_input, output_line);
  if (type_ == NT_TANH) {
    FuncInplace<GFunc>(no_, output_line);
  } else if (type_ == NT_LOGISTIC) {
    FuncInplace<FFunc>(no_, output_line);
  } else if (type_ == NT_POSCLIP) {
    FuncInplace<ClipFFunc>(no_, output_line);
  } else if (type_ == NT_SYMCLIP) {
    FuncInplace<ClipGFunc>(no_, output_line);
  } else if (type_ == NT_RELU) {
    FuncInplace<Relu>(no_, output_line);
  } else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC) {
    SoftmaxInPlace(no_, output_line);
  } else if (type_ != NT_LINEAR) {
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  }
}

// Runs backward propagation of errors on the deltas line.
// See NetworkCpp for a detailed discussion of the arguments.
bool FullyConnected::Backward(bool debug, const NetworkIO& fwd_deltas,
                              NetworkScratch* scratch,
                              NetworkIO* back_deltas) {
  if (debug) DisplayBackward(fwd_deltas);
  back_deltas->Resize(fwd_deltas, ni_);
  GenericVector<NetworkScratch::FloatVec> errors;
  errors.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < errors.size(); ++i) errors[i].Init(no_, scratch);
  GenericVector<NetworkScratch::FloatVec> temp_backprops;
  if (needs_to_backprop_) {
    temp_backprops.init_to_size(kNumThreads, NetworkScratch::FloatVec());
    for (int i = 0; i < kNumThreads; ++i) temp_backprops[i].Init(ni_, scratch);
  }
  int width = fwd_deltas.Width();
  NetworkScratch::GradientStore errors_t;
  errors_t.Init(no_, width, scratch);
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    int thread_id = 0;
#endif
    double* backprop = NULL;
    if (needs_to_backprop_) backprop = temp_backprops[thread_id];
    double* curr_errors = errors[thread_id];
    BackwardTimeStep(fwd_deltas, t, curr_errors, errors_t.get(), backprop);
    if (backprop != NULL) {
      back_deltas->WriteTimeStep(t, backprop);
    }
  }
  FinishBackward(*errors_t.get());
  if (needs_to_backprop_) {
    back_deltas->ZeroInvalidElements();
    back_deltas->CopyWithNormalization(*back_deltas, fwd_deltas);
#if DEBUG_DETAIL > 0
    tprintf("F Backprop:%s\n", name_.string());
    back_deltas->Print(10);
#endif
    return true;
  }
  return false;  // No point going further back.
}

void FullyConnected::BackwardTimeStep(const NetworkIO& fwd_deltas, int t,
                                      double* curr_errors,
                                      TransposedArray* errors_t,
                                      double* backprop) {
  if (type_ == NT_TANH)
    acts_.FuncMultiply<GPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_LOGISTIC)
    acts_.FuncMultiply<FPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_POSCLIP)
    acts_.FuncMultiply<ClipFPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SYMCLIP)
    acts_.FuncMultiply<ClipGPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_RELU)
    acts_.FuncMultiply<ReluPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC ||
           type_ == NT_LINEAR)
    fwd_deltas.ReadTimeStep(t, curr_errors);  // fwd_deltas are the errors.
  else
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  // Generate backprop only if needed by the lower layer.
  if (backprop != NULL) weights_.VectorDotMatrix(curr_errors, backprop);
  errors_t->WriteStrided(t, curr_errors);
}

void FullyConnected::FinishBackward(const TransposedArray& errors_t) {
  if (external_source_ == NULL)
    weights_.SumOuterTransposed(errors_t, source_t_, true);
  else
    weights_.SumOuterTransposed(errors_t, *external_source_, true);
}

// Updates the weights using the given learning rate and momentum.
// num_samples is the quotient to be used in the adagrad computation iff
// use_ada_grad_ is true.
void FullyConnected::Update(float learning_rate, float momentum,
                            int num_samples) {
  weights_.Update(learning_rate, momentum, num_samples);
}

// Sums the products of weight updates in *this and other, splitting into
// positive (same direction) in *same and negative (different direction) in
// *changed.
void FullyConnected::CountAlternators(const Network& other, double* same,
                                      double* changed) const {
  ASSERT_HOST(other.type() == type_);
  const FullyConnected* fc = static_cast<const FullyConnected*>(&other);
  weights_.CountAlternators(fc->weights_, same, changed);
}

}  // namespace tesseract.
<commit_msg>Optimize LSTM code for builds without OpenMP<commit_after>///////////////////////////////////////////////////////////////////////
// File:        fullyconnected.cpp
// Description: Simple feed-forward layer with various non-linearities.
// Author:      Ray Smith
// Created:     Wed Feb 26 14:49:15 PST 2014
//
// (C) Copyright 2014, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
///////////////////////////////////////////////////////////////////////

#include "fullyconnected.h"

#ifdef _OPENMP
#include <omp.h>
#endif
#include <stdio.h>
#include <stdlib.h>

#include "functions.h"
#include "networkscratch.h"

// Number of threads to use for parallel calculation of Forward and Backward.
#ifdef _OPENMP
const int kNumThreads = 4;
#else
const int kNumThreads = 1;
#endif

namespace tesseract {

FullyConnected::FullyConnected(const STRING& name, int ni, int no,
                               NetworkType type)
  : Network(type, name, ni, no), external_source_(NULL), int_mode_(false) {
}

FullyConnected::~FullyConnected() {
}

// Returns the shape output from the network given an input shape (which may
// be partially unknown ie zero).
StaticShape FullyConnected::OutputShape(const StaticShape& input_shape) const {
  LossType loss_type = LT_NONE;
  if (type_ == NT_SOFTMAX)
    loss_type = LT_CTC;
  else if (type_ == NT_SOFTMAX_NO_CTC)
    loss_type = LT_SOFTMAX;
  else if (type_ == NT_LOGISTIC)
    loss_type = LT_LOGISTIC;
  StaticShape result(input_shape);
  result.set_depth(no_);
  result.set_loss_type(loss_type);
  return result;
}

// Suspends/Enables training by setting the training_ flag.
void FullyConnected::SetEnableTraining(TrainingState state) {
  if (state == TS_RE_ENABLE) {
    // Enable only from temp disabled.
    if (training_ == TS_TEMP_DISABLE) training_ = TS_ENABLED;
  } else if (state == TS_TEMP_DISABLE) {
    // Temp disable only from enabled.
    if (training_ == TS_ENABLED) training_ = state;
  } else {
    if (state == TS_ENABLED && training_ != TS_ENABLED)
      weights_.InitBackward();
    training_ = state;
  }
}

// Sets up the network for training. Initializes weights using weights of
// scale `range` picked according to the random number generator `randomizer`.
int FullyConnected::InitWeights(float range, TRand* randomizer) {
  Network::SetRandomizer(randomizer);
  num_weights_ = weights_.InitWeightsFloat(no_, ni_ + 1, TestFlag(NF_ADA_GRAD),
                                           range, randomizer);
  return num_weights_;
}

// Converts a float network to an int network.
void FullyConnected::ConvertToInt() {
  weights_.ConvertToInt();
}

// Provides debug output on the weights.
void FullyConnected::DebugWeights() {
  weights_.Debug2D(name_.string());
}

// Writes to the given file. Returns false in case of error.
bool FullyConnected::Serialize(TFile* fp) const {
  if (!Network::Serialize(fp)) return false;
  if (!weights_.Serialize(IsTraining(), fp)) return false;
  return true;
}

// Reads from the given file. Returns false in case of error.
bool FullyConnected::DeSerialize(TFile* fp) {
  return weights_.DeSerialize(IsTraining(), fp);
}

// Runs forward propagation of activations on the input line.
// See NetworkCpp for a detailed discussion of the arguments.
void FullyConnected::Forward(bool debug, const NetworkIO& input,
                             const TransposedArray* input_transpose,
                             NetworkScratch* scratch, NetworkIO* output) {
  int width = input.Width();
  if (type_ == NT_SOFTMAX)
    output->ResizeFloat(input, no_);
  else
    output->Resize(input, no_);
  SetupForward(input, input_transpose);
  GenericVector<NetworkScratch::FloatVec> temp_lines;
  temp_lines.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  GenericVector<NetworkScratch::FloatVec> curr_input;
  curr_input.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < kNumThreads; ++i) {
    temp_lines[i].Init(no_, scratch);
    curr_input[i].Init(ni_, scratch);
  }
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = 0;
#endif
    double* temp_line = temp_lines[thread_id];
    const double* d_input = NULL;
    const inT8* i_input = NULL;
    if (input.int_mode()) {
      i_input = input.i(t);
    } else {
      input.ReadTimeStep(t, curr_input[thread_id]);
      d_input = curr_input[thread_id];
    }
    ForwardTimeStep(d_input, i_input, t, temp_line);
    output->WriteTimeStep(t, temp_line);
    if (IsTraining() && type_ != NT_SOFTMAX) {
      acts_.CopyTimeStepFrom(t, *output, t);
    }
  }
  // Zero all the elements that are in the padding around images that allows
  // multiple different-sized images to exist in a single array.
  // acts_ is only used if this is not a softmax op.
  if (IsTraining() && type_ != NT_SOFTMAX) {
    acts_.ZeroInvalidElements();
  }
  output->ZeroInvalidElements();
#if DEBUG_DETAIL > 0
  tprintf("F Output:%s\n", name_.string());
  output->Print(10);
#endif
  if (debug) DisplayForward(*output);
}

// Components of Forward so FullyConnected can be reused inside LSTM.
void FullyConnected::SetupForward(const NetworkIO& input,
                                  const TransposedArray* input_transpose) {
  // Softmax output is always float, so save the input type.
  int_mode_ = input.int_mode();
  if (IsTraining()) {
    acts_.Resize(input, no_);
    // Source_ is a transposed copy of input. It isn't needed if provided.
    external_source_ = input_transpose;
    if (external_source_ == NULL) source_t_.ResizeNoInit(ni_, input.Width());
  }
}

void FullyConnected::ForwardTimeStep(const double* d_input, const inT8* i_input,
                                     int t, double* output_line) {
  // input is copied to source_ line-by-line for cache coherency.
  if (IsTraining() && external_source_ == NULL && d_input != NULL)
    source_t_.WriteStrided(t, d_input);
  if (d_input != NULL)
    weights_.MatrixDotVector(d_input, output_line);
  else
    weights_.MatrixDotVector(i_input, output_line);
  if (type_ == NT_TANH) {
    FuncInplace<GFunc>(no_, output_line);
  } else if (type_ == NT_LOGISTIC) {
    FuncInplace<FFunc>(no_, output_line);
  } else if (type_ == NT_POSCLIP) {
    FuncInplace<ClipFFunc>(no_, output_line);
  } else if (type_ == NT_SYMCLIP) {
    FuncInplace<ClipGFunc>(no_, output_line);
  } else if (type_ == NT_RELU) {
    FuncInplace<Relu>(no_, output_line);
  } else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC) {
    SoftmaxInPlace(no_, output_line);
  } else if (type_ != NT_LINEAR) {
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  }
}

// Runs backward propagation of errors on the deltas line.
// See NetworkCpp for a detailed discussion of the arguments.
bool FullyConnected::Backward(bool debug, const NetworkIO& fwd_deltas,
                              NetworkScratch* scratch,
                              NetworkIO* back_deltas) {
  if (debug) DisplayBackward(fwd_deltas);
  back_deltas->Resize(fwd_deltas, ni_);
  GenericVector<NetworkScratch::FloatVec> errors;
  errors.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < kNumThreads; ++i) errors[i].Init(no_, scratch);
  GenericVector<NetworkScratch::FloatVec> temp_backprops;
  if (needs_to_backprop_) {
    temp_backprops.init_to_size(kNumThreads, NetworkScratch::FloatVec());
    for (int i = 0; i < kNumThreads; ++i) temp_backprops[i].Init(ni_, scratch);
  }
  int width = fwd_deltas.Width();
  NetworkScratch::GradientStore errors_t;
  errors_t.Init(no_, width, scratch);
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    int thread_id = 0;
#endif
    double* backprop = NULL;
    if (needs_to_backprop_) backprop = temp_backprops[thread_id];
    double* curr_errors = errors[thread_id];
    BackwardTimeStep(fwd_deltas, t, curr_errors, errors_t.get(), backprop);
    if (backprop != NULL) {
      back_deltas->WriteTimeStep(t, backprop);
    }
  }
  FinishBackward(*errors_t.get());
  if (needs_to_backprop_) {
    back_deltas->ZeroInvalidElements();
    back_deltas->CopyWithNormalization(*back_deltas, fwd_deltas);
#if DEBUG_DETAIL > 0
    tprintf("F Backprop:%s\n", name_.string());
    back_deltas->Print(10);
#endif
    return true;
  }
  return false;  // No point going further back.
}

void FullyConnected::BackwardTimeStep(const NetworkIO& fwd_deltas, int t,
                                      double* curr_errors,
                                      TransposedArray* errors_t,
                                      double* backprop) {
  if (type_ == NT_TANH)
    acts_.FuncMultiply<GPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_LOGISTIC)
    acts_.FuncMultiply<FPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_POSCLIP)
    acts_.FuncMultiply<ClipFPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SYMCLIP)
    acts_.FuncMultiply<ClipGPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_RELU)
    acts_.FuncMultiply<ReluPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC ||
           type_ == NT_LINEAR)
    fwd_deltas.ReadTimeStep(t, curr_errors);  // fwd_deltas are the errors.
  else
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  // Generate backprop only if needed by the lower layer.
  if (backprop != NULL) weights_.VectorDotMatrix(curr_errors, backprop);
  errors_t->WriteStrided(t, curr_errors);
}

void FullyConnected::FinishBackward(const TransposedArray& errors_t) {
  if (external_source_ == NULL)
    weights_.SumOuterTransposed(errors_t, source_t_, true);
  else
    weights_.SumOuterTransposed(errors_t, *external_source_, true);
}

// Updates the weights using the given learning rate and momentum.
// num_samples is the quotient to be used in the adagrad computation iff
// use_ada_grad_ is true.
void FullyConnected::Update(float learning_rate, float momentum,
                            int num_samples) {
  weights_.Update(learning_rate, momentum, num_samples);
}

// Sums the products of weight updates in *this and other, splitting into
// positive (same direction) in *same and negative (different direction) in
// *changed.
void FullyConnected::CountAlternators(const Network& other, double* same,
                                      double* changed) const {
  ASSERT_HOST(other.type() == type_);
  const FullyConnected* fc = static_cast<const FullyConnected*>(&other);
  weights_.CountAlternators(fc->weights_, same, changed);
}

}  // namespace tesseract.
<|endoftext|>