<commit_before>import json
import requests
import six

from craftai import helpers
from craftai.constants import AGENT_ID_PATTERN
from craftai.errors import CraftAiCredentialsError, CraftAiBadRequestError, CraftAiNotFoundError
from craftai.errors import CraftAiUnknownError, CraftAiInternalError
from craftai.interpreter import Interpreter
from craftai.jwt_decode import jwt_decode

class CraftAIClient(object):
  """Client class for craft ai's API"""

  def __init__(self, cfg):
    self._base_url = ""
    self._headers = {}
    self._config = {}

    try:
      self.config = cfg
    except (CraftAiCredentialsError, CraftAiBadRequestError) as e:
      raise e

  @property
  def config(self):
    return self._config

  @config.setter
  def config(self, cfg):
    cfg = cfg.copy()
    (payload, _, _, _) = jwt_decode(cfg.get("token"))
    cfg["owner"] = cfg["owner"] if "owner" in cfg else payload.get("owner")
    cfg["project"] = cfg["project"] if "project" in cfg else payload.get("project")
    cfg["url"] = cfg["url"] if "url" in cfg else payload.get("platform")

    if not isinstance(cfg.get("project"), six.string_types):
      raise CraftAiCredentialsError("""Unable to create client with no"""
                                    """ or invalid project provided.""")
    else:
      splitted_project = cfg.get("project").split("/")
      if len(splitted_project) == 2:
        cfg["owner"] = splitted_project[0]
        cfg["project"] = splitted_project[1]
      elif len(splitted_project) > 2:
        raise CraftAiCredentialsError("""Unable to create client with invalid"""
                                      """ project name.""")
    if not isinstance(cfg.get("owner"), six.string_types):
      raise CraftAiCredentialsError("""Unable to create client with no"""
                                    """ or invalid owner provided.""")
    if not isinstance(cfg.get("operationsChunksSize"), six.integer_types):
      cfg["operationsChunksSize"] = 200
    if not isinstance(cfg.get("url"), six.string_types):
      cfg["url"] = "https://beta.craft.ai"
    if cfg.get("url").endswith("/"):
      raise CraftAiBadRequestError("""Unable to create client with"""
                                   """ invalid url provided. The url"""
                                   """ should not terminate with a"""
                                   """ slash.""")
    self._config = cfg

    self._base_url = "{}/api/v1/{}/{}".format(self.config["url"],
                                              self.config["owner"],
                                              self.config["project"])

    # Headers have to be reset here to avoid multiple definitions
    # of the 'Authorization' header if config is modified
    self._headers = {}
    self._headers["Authorization"] = "Bearer " + self.config.get("token")

  #################
  # Agent methods #
  #################

  def create_agent(self, configuration, agent_id=""):
    # Building final headers
    ct_header = {"Content-Type": "application/json; charset=utf-8"}
    headers = helpers.join_dicts(self._headers, ct_header)

    # Building payload and checking that it is valid for a JSON
    # serialization
    payload = {"configuration": configuration}

    if agent_id != "":
      # Raises an error when agent_id is invalid
      self._check_agent_id(agent_id)

      payload["id"] = agent_id

    try:
      json_pl = json.dumps(payload)
    except TypeError as e:
      raise CraftAiBadRequestError("Invalid configuration or agent id given. {}"
                                   .format(e.__str__()))

    req_url = "{}/agents".format(self._base_url)
    resp = requests.post(req_url, headers=headers, data=json_pl)

    agent = self._decode_response(resp)

    return agent

  def get_agent(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}".format(self._base_url, agent_id)
    resp = requests.get(req_url, headers=headers)

    agent = self._decode_response(resp)

    return agent

  def list_agents(self):
    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents".format(self._base_url)
    resp = requests.get(req_url, headers=headers)

    agents = self._decode_response(resp)

    return agents["agentsList"]

  def delete_agent(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}".format(self._base_url, agent_id)
    resp = requests.delete(req_url, headers=headers)

    decoded_resp = self._decode_response(resp)

    return decoded_resp

  def get_shared_agent_inspector_url(self, agent_id, timestamp=None):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}/shared".format(self._base_url, agent_id)
    resp = requests.get(req_url, headers=headers)

    url = self._decode_response(resp)

    if timestamp != None:
      return "{}?t={}".format(url["shortUrl"], str(timestamp))

    return url["shortUrl"]

  def delete_shared_agent_inspector_url(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}/shared".format(self._base_url, agent_id)
    resp = requests.delete(req_url, headers=headers)

    decoded_resp = self._decode_response(resp)

    return decoded_resp

  ###################
  # Context methods #
  ###################

  def add_operations(self, agent_id, operations):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # Building final headers
    ct_header = {"Content-Type": "application/json; charset=utf-8"}
    headers = helpers.join_dicts(self._headers, ct_header)

    session = requests.Session()
    offset = 0

    while True:
      next_offset = offset + self.config["operationsChunksSize"]

      try:
        json_pl = json.dumps(operations[offset:next_offset])
      except TypeError as e:
        raise CraftAiBadRequestError("Invalid configuration or agent id given. {}"
                                     .format(e.__str__()))

      req_url = "{}/agents/{}/context".format(self._base_url, agent_id)
      resp = session.post(req_url, headers=headers, data=json_pl)

      self._decode_response(resp)

      if next_offset >= len(operations):
        return {
          "message": "Successfully added %i operation(s) to the agent \"%s/%s/%s\" context."
                     % (len(operations), self.config["owner"], self.config["project"], agent_id)
        }

      offset = next_offset

  def _get_operations_list_pages(self, url, ops_list):
    if url is None:
      return ops_list

    headers = self._headers.copy()

    resp = requests.get(url, headers=headers)

    new_ops_list = self._decode_response(resp)
    next_page_url = resp.headers.get("x-craft-ai-next-page-url")

    return self._get_operations_list_pages(next_page_url, ops_list + new_ops_list)

  def get_operations_list(self, agent_id, start=None, end=None):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/context".format(self._base_url, agent_id)
    req_params = {
      "start": start,
      "end": end
    }
    resp = requests.get(req_url, params=req_params, headers=headers)

    initial_ops_list = self._decode_response(resp)
    next_page_url = resp.headers.get("x-craft-ai-next-page-url")

    return self._get_operations_list_pages(next_page_url, initial_ops_list)

  def get_context_state(self, agent_id, timestamp):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/context/state?t={}".format(self._base_url,
                                                       agent_id,
                                                       timestamp)
    resp = requests.get(req_url, headers=headers)

    context_state = self._decode_response(resp)

    return context_state

  #########################
  # Decision tree methods #
  #########################

  def get_decision_tree(self, agent_id, timestamp):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/decision/tree?t={}".format(self._base_url,
                                                       agent_id,
                                                       timestamp)

    resp = requests.get(req_url, headers=headers)

    decision_tree = self._decode_response(resp)

    return decision_tree

  @staticmethod
  def decide(tree, *args):
    return Interpreter.decide(tree, args)

  @staticmethod
  def _decode_response(response):
    # https://github.com/kennethreitz/requests/blob/master/requests/status_codes.py
    if response.status_code == requests.codes.not_found:
      raise CraftAiNotFoundError(response.text)
    if response.status_code == requests.codes.bad_request:
      raise CraftAiBadRequestError(response.text)
    if response.status_code == requests.codes.unauthorized:
      raise CraftAiCredentialsError(response.text)
    if response.status_code == requests.codes.request_timeout:
      raise CraftAiBadRequestError("Request has timed out")
    if response.status_code == requests.codes.gateway_timeout:
      raise CraftAiInternalError("Response has timed out")

    try:
      return response.json()
    except:
      raise CraftAiUnknownError(response.text)

  @staticmethod
  def _check_agent_id(agent_id):
    """Checks that the given agent_id is a valid non-empty string.

    Raises an error if the given agent_id is not of type string or if it is
    an empty string.
    """
    if (not isinstance(agent_id, six.string_types) or
        AGENT_ID_PATTERN.match(agent_id) is None):
      raise CraftAiBadRequestError("""Invalid agent id given."""
                                   """It must be a string containaing only"""
                                   """characters in \"a-zA-Z0-9_-\""""
                                   """and must be between 1 and 36 characters.""")
<commit_msg>Specific user agent for the requests to craft ai<commit_after>import json
from platform import python_implementation, python_version
import requests
import six

from craftai import helpers, __version__ as pkg_version
from craftai.constants import AGENT_ID_PATTERN
from craftai.errors import CraftAiCredentialsError, CraftAiBadRequestError, CraftAiNotFoundError
from craftai.errors import CraftAiUnknownError, CraftAiInternalError
from craftai.interpreter import Interpreter
from craftai.jwt_decode import jwt_decode

USER_AGENT = "craft-ai-client-python/{} [{} {}]".format(pkg_version,
                                                        python_implementation(),
                                                        python_version())

class CraftAIClient(object):
  """Client class for craft ai's API"""

  def __init__(self, cfg):
    self._base_url = ""
    self._headers = {}
    self._config = {}

    try:
      self.config = cfg
    except (CraftAiCredentialsError, CraftAiBadRequestError) as e:
      raise e

  @property
  def config(self):
    return self._config

  @config.setter
  def config(self, cfg):
    cfg = cfg.copy()
    (payload, _, _, _) = jwt_decode(cfg.get("token"))
    cfg["owner"] = cfg["owner"] if "owner" in cfg else payload.get("owner")
    cfg["project"] = cfg["project"] if "project" in cfg else payload.get("project")
    cfg["url"] = cfg["url"] if "url" in cfg else payload.get("platform")

    if not isinstance(cfg.get("project"), six.string_types):
      raise CraftAiCredentialsError("""Unable to create client with no"""
                                    """ or invalid project provided.""")
    else:
      splitted_project = cfg.get("project").split("/")
      if len(splitted_project) == 2:
        cfg["owner"] = splitted_project[0]
        cfg["project"] = splitted_project[1]
      elif len(splitted_project) > 2:
        raise CraftAiCredentialsError("""Unable to create client with invalid"""
                                      """ project name.""")
    if not isinstance(cfg.get("owner"), six.string_types):
      raise CraftAiCredentialsError("""Unable to create client with no"""
                                    """ or invalid owner provided.""")
    if not isinstance(cfg.get("operationsChunksSize"), six.integer_types):
      cfg["operationsChunksSize"] = 200
    if not isinstance(cfg.get("url"), six.string_types):
      cfg["url"] = "https://beta.craft.ai"
    if cfg.get("url").endswith("/"):
      raise CraftAiBadRequestError("""Unable to create client with"""
                                   """ invalid url provided. The url"""
                                   """ should not terminate with a"""
                                   """ slash.""")
    self._config = cfg

    self._base_url = "{}/api/v1/{}/{}".format(self.config["url"],
                                              self.config["owner"],
                                              self.config["project"])

    # Headers have to be reset here to avoid multiple definitions
    # of the 'Authorization' header if config is modified
    self._headers = {}
    self._headers["Authorization"] = "Bearer " + self.config.get("token")
    self._headers["User-Agent"] = USER_AGENT

  #################
  # Agent methods #
  #################

  def create_agent(self, configuration, agent_id=""):
    # Building final headers
    ct_header = {"Content-Type": "application/json; charset=utf-8"}
    headers = helpers.join_dicts(self._headers, ct_header)

    # Building payload and checking that it is valid for a JSON
    # serialization
    payload = {"configuration": configuration}

    if agent_id != "":
      # Raises an error when agent_id is invalid
      self._check_agent_id(agent_id)

      payload["id"] = agent_id

    try:
      json_pl = json.dumps(payload)
    except TypeError as e:
      raise CraftAiBadRequestError("Invalid configuration or agent id given. {}"
                                   .format(e.__str__()))

    req_url = "{}/agents".format(self._base_url)
    resp = requests.post(req_url, headers=headers, data=json_pl)

    agent = self._decode_response(resp)

    return agent

  def get_agent(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}".format(self._base_url, agent_id)
    resp = requests.get(req_url, headers=headers)

    agent = self._decode_response(resp)

    return agent

  def list_agents(self):
    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents".format(self._base_url)
    resp = requests.get(req_url, headers=headers)

    agents = self._decode_response(resp)

    return agents["agentsList"]

  def delete_agent(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}".format(self._base_url, agent_id)
    resp = requests.delete(req_url, headers=headers)

    decoded_resp = self._decode_response(resp)

    return decoded_resp

  def get_shared_agent_inspector_url(self, agent_id, timestamp=None):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}/shared".format(self._base_url, agent_id)
    resp = requests.get(req_url, headers=headers)

    url = self._decode_response(resp)

    if timestamp != None:
      return "{}?t={}".format(url["shortUrl"], str(timestamp))

    return url["shortUrl"]

  def delete_shared_agent_inspector_url(self, agent_id):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # No supplementary headers needed
    headers = self._headers.copy()

    req_url = "{}/agents/{}/shared".format(self._base_url, agent_id)
    resp = requests.delete(req_url, headers=headers)

    decoded_resp = self._decode_response(resp)

    return decoded_resp

  ###################
  # Context methods #
  ###################

  def add_operations(self, agent_id, operations):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    # Building final headers
    ct_header = {"Content-Type": "application/json; charset=utf-8"}
    headers = helpers.join_dicts(self._headers, ct_header)

    session = requests.Session()
    offset = 0

    while True:
      next_offset = offset + self.config["operationsChunksSize"]

      try:
        json_pl = json.dumps(operations[offset:next_offset])
      except TypeError as e:
        raise CraftAiBadRequestError("Invalid configuration or agent id given. {}"
                                     .format(e.__str__()))

      req_url = "{}/agents/{}/context".format(self._base_url, agent_id)
      resp = session.post(req_url, headers=headers, data=json_pl)

      self._decode_response(resp)

      if next_offset >= len(operations):
        return {
          "message": "Successfully added %i operation(s) to the agent \"%s/%s/%s\" context."
                     % (len(operations), self.config["owner"], self.config["project"], agent_id)
        }

      offset = next_offset

  def _get_operations_list_pages(self, url, ops_list):
    if url is None:
      return ops_list

    headers = self._headers.copy()

    resp = requests.get(url, headers=headers)

    new_ops_list = self._decode_response(resp)
    next_page_url = resp.headers.get("x-craft-ai-next-page-url")

    return self._get_operations_list_pages(next_page_url, ops_list + new_ops_list)

  def get_operations_list(self, agent_id, start=None, end=None):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/context".format(self._base_url, agent_id)
    req_params = {
      "start": start,
      "end": end
    }
    resp = requests.get(req_url, params=req_params, headers=headers)

    initial_ops_list = self._decode_response(resp)
    next_page_url = resp.headers.get("x-craft-ai-next-page-url")

    return self._get_operations_list_pages(next_page_url, initial_ops_list)

  def get_context_state(self, agent_id, timestamp):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/context/state?t={}".format(self._base_url,
                                                       agent_id,
                                                       timestamp)
    resp = requests.get(req_url, headers=headers)

    context_state = self._decode_response(resp)

    return context_state

  #########################
  # Decision tree methods #
  #########################

  def get_decision_tree(self, agent_id, timestamp):
    # Raises an error when agent_id is invalid
    self._check_agent_id(agent_id)

    headers = self._headers.copy()

    req_url = "{}/agents/{}/decision/tree?t={}".format(self._base_url,
                                                       agent_id,
                                                       timestamp)

    resp = requests.get(req_url, headers=headers)

    decision_tree = self._decode_response(resp)

    return decision_tree

  @staticmethod
  def decide(tree, *args):
    return Interpreter.decide(tree, args)

  @staticmethod
  def _decode_response(response):
    # https://github.com/kennethreitz/requests/blob/master/requests/status_codes.py
    if response.status_code == requests.codes.not_found:
      raise CraftAiNotFoundError(response.text)
    if response.status_code == requests.codes.bad_request:
      raise CraftAiBadRequestError(response.text)
    if response.status_code == requests.codes.unauthorized:
      raise CraftAiCredentialsError(response.text)
    if response.status_code == requests.codes.request_timeout:
      raise CraftAiBadRequestError("Request has timed out")
    if response.status_code == requests.codes.gateway_timeout:
      raise CraftAiInternalError("Response has timed out")

    try:
      return response.json()
    except:
      raise CraftAiUnknownError(response.text)

  @staticmethod
  def _check_agent_id(agent_id):
    """Checks that the given agent_id is a valid non-empty string.

    Raises an error if the given agent_id is not of type string or if it is
    an empty string.
    """
    if (not isinstance(agent_id, six.string_types) or
        AGENT_ID_PATTERN.match(agent_id) is None):
      raise CraftAiBadRequestError("""Invalid agent id given."""
                                   """It must be a string containaing only"""
                                   """characters in \"a-zA-Z0-9_-\""""
                                   """and must be between 1 and 36 characters.""")
<|endoftext|>


<commit_before># Python imports.
from collections import defaultdict
import random

# Other imports.
from simple_rl.mdp.StateClass import State

class Option(object):

	def __init__(self, init_predicate, term_predicate, policy, name="o", term_prob=0.01):
		'''
		Args:
			init_func (S --> {0,1})
			init_func (S --> {0,1})
			policy (S --> A)
		'''
		self.init_predicate = init_predicate
		self.term_predicate = term_predicate
		self.term_flag = False
		self.name = name
		self.term_prob = term_prob

		if type(policy) is defaultdict or type(policy) is dict:
			self.policy_dict = dict(policy)
			self.policy = self.policy_from_dict
		else:
			self.policy = policy

	def is_init_true(self, ground_state):
		return self.init_predicate.is_true(ground_state)

	def is_term_true(self, ground_state):
		return self.term_predicate.is_true(ground_state) or self.term_flag or self.term_prob > random.random()

	def act(self, ground_state):
		return self.policy(ground_state)

	def set_policy(self, policy):
		self.policy = policy

	def set_name(self, new_name):
		self.name = new_name

	def act_until_terminal(self, cur_state, transition_func):
		'''
		Summary:
			Executes the option until termination.
		'''
		if self.is_init_true(cur_state):
			cur_state = transition_func(cur_state, self.act(cur_state))
			while not self.is_term_true(cur_state):
				cur_state = transition_func(cur_state, self.act(cur_state))

		return cur_state

	def rollout(self, cur_state, reward_func, transition_func, step_cost=0):
		'''
		Summary:
			Executes the option until termination.

		Returns:
			(tuple):
				1. (State): state we landed in.
				2. (float): Reward from the trajectory.
		'''
		total_reward = 0
		if self.is_init_true(cur_state):
			# First step.
			total_reward += reward_func(cur_state, self.act(cur_state)) - step_cost
			cur_state = transition_func(cur_state, self.act(cur_state))

			# Act until terminal.
			while not self.is_term_true(cur_state):
				cur_state = transition_func(cur_state, self.act(cur_state))
				total_reward += reward_func(cur_state, self.act(cur_state)) - step_cost

		return cur_state, total_reward

	def policy_from_dict(self, state):
		if state not in self.policy_dict.keys():
			self.term_flag = True
			return random.choice(list(set(self.policy_dict.values())))
		else:
			self.term_flag = False
			return self.policy_dict[state]

	def term_func_from_list(self, state):
		return state in self.term_list

	def __str__(self):
		return "option." + str(self.name)<commit_msg>rollout depth added to option<commit_after># Python imports.
from collections import defaultdict
import random

# Other imports.
from simple_rl.mdp.StateClass import State

class Option(object):

	def __init__(self, init_predicate, term_predicate, policy, name="o", term_prob=0.00):
		'''
		Args:
			init_func (S --> {0,1})
			init_func (S --> {0,1})
			policy (S --> A)
		'''
		self.init_predicate = init_predicate
		self.term_predicate = term_predicate
		self.term_flag = False
		self.name = name
		self.term_prob = term_prob

		if type(policy) is defaultdict or type(policy) is dict:
			self.policy_dict = dict(policy)
			self.policy = self.policy_from_dict
		else:
			self.policy = policy

	def is_init_true(self, ground_state):
		return self.init_predicate.is_true(ground_state)

	def is_term_true(self, ground_state):
		return self.term_predicate.is_true(ground_state) or self.term_flag or self.term_prob > random.random()

	def act(self, ground_state):
		return self.policy(ground_state)

	def set_policy(self, policy):
		self.policy = policy

	def set_name(self, new_name):
		self.name = new_name

	def act_until_terminal(self, cur_state, transition_func):
		'''
		Summary:
			Executes the option until termination.
		'''
		if self.is_init_true(cur_state):
			cur_state = transition_func(cur_state, self.act(cur_state))
			while not self.is_term_true(cur_state):
				cur_state = transition_func(cur_state, self.act(cur_state))

		return cur_state

	def rollout(self, cur_state, reward_func, transition_func, max_rollout_depth, step_cost=0):
		'''
		Summary:
			Executes the option until termination.

		Args:
			cur_state (simple_rl.State)
			reward_func (lambda)
			transition_func (lambda)
			max_rollout_depth (int)
			step_cost (float)

		Returns:
			(tuple):
				1. (State): state we landed in.
				2. (float): Reward from the trajectory.
		'''
		total_reward = 0
		rollout_depth = 0

		if self.is_init_true(cur_state):
			# First step.
			total_reward += reward_func(cur_state, self.act(cur_state)) - step_cost
			cur_state = transition_func(cur_state, self.act(cur_state))

			# Act until terminal.
			while rollout_depth < max_rollout_depth and not self.is_term_true(cur_state):
				cur_state = transition_func(cur_state, self.act(cur_state))
				total_reward += reward_func(cur_state, self.act(cur_state)) - step_cost
				rollout_depth += 1

		# print self.name, cur_state
		return cur_state, total_reward

	def policy_from_dict(self, state):
		if state not in self.policy_dict.keys():
			self.term_flag = True
			return random.choice(list(set(self.policy_dict.values())))
		else:
			self.term_flag = False
			return self.policy_dict[state]

	def term_func_from_list(self, state):
		return state in self.term_list

	def __str__(self):
		return "option." + str(self.name)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# google-cloud-datacatalog documentation build configuration file
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(".."))

__version__ = "0.1.0"

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = "1.6.3"

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.autosummary",
    "sphinx.ext.intersphinx",
    "sphinx.ext.coverage",
    "sphinx.ext.napoleon",
    "sphinx.ext.todo",
    "sphinx.ext.viewcode",
]

# autodoc/autosummary flags
autoclass_content = "both"
autodoc_default_flags = ["members"]
autosummary_generate = True

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# Allow markdown includes (so releases.md can include CHANGLEOG.md)
# http://www.sphinx-doc.org/en/master/markdown.html
source_parsers = {".md": "recommonmark.parser.CommonMarkParser"}

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = [".rst", ".md"]

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = "index"

# General information about the project.
project = u"google-cloud-datacatalog"
copyright = u"2017, Google"
author = u"Google APIs"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The full version, including alpha/beta/rc tags.
release = __version__
# The short X.Y version.
version = ".".join(release.split(".")[0:2])

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["_build"]

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = "alabaster"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    "description": "Google Cloud Client Libraries for Python",
    "github_user": "googleapis",
    "github_repo": "google-cloud-python",
    "github_banner": True,
    "font_family": "'Roboto', Georgia, sans",
    "head_font_family": "'Roboto', Georgia, serif",
    "code_font_family": "'Roboto Mono', 'Consolas', monospace",
}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = []

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = "google-cloud-datacatalog-doc"

# -- Options for warnings ------------------------------------------------------

suppress_warnings = [
    # Temporarily suppress this to avoid "more than one target found for
    # cross-reference" warning, which are intractable for us to avoid while in
    # a mono-repo.
    # See https://github.com/sphinx-doc/sphinx/blob
    # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843
    "ref.python"
]

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',
    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',
    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (
        master_doc,
        "google-cloud-datacatalog.tex",
        u"google-cloud-datacatalog Documentation",
        author,
        "manual",
    )
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (
        master_doc,
        "google-cloud-datacatalog",
        u"google-cloud-datacatalog Documentation",
        [author],
        1,
    )
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (
        master_doc,
        "google-cloud-datacatalog",
        u"google-cloud-datacatalog Documentation",
        author,
        "google-cloud-datacatalog",
        "GAPIC library for the {metadata.shortName} v1beta1 service",
        "APIs",
    )
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    "python": ("http://python.readthedocs.org/en/latest/", None),
    "gax": ("https://gax-python.readthedocs.org/en/latest/", None),
    "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
    "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None),
    "google.api_core": (
        "https://googleapis.github.io/google-cloud-python/latest",
        None,
    ),
    "grpc": ("https://grpc.io/grpc/python/", None),
    "requests": ("http://docs.python-requests.org/en/master/", None),
    "fastavro": ("https://fastavro.readthedocs.io/en/stable/", None),
    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
}

# Napoleon settings
napoleon_google_docstring = True
napoleon_numpy_docstring = True
napoleon_include_private_with_doc = False
napoleon_include_special_with_doc = True
napoleon_use_admonition_for_examples = False
napoleon_use_admonition_for_notes = False
napoleon_use_admonition_for_references = False
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = True
<commit_msg>Update intersphinx mapping for requests. (#8805)<commit_after># -*- coding: utf-8 -*-
#
# google-cloud-datacatalog documentation build configuration file
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(".."))

__version__ = "0.1.0"

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = "1.6.3"

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.autosummary",
    "sphinx.ext.intersphinx",
    "sphinx.ext.coverage",
    "sphinx.ext.napoleon",
    "sphinx.ext.todo",
    "sphinx.ext.viewcode",
]

# autodoc/autosummary flags
autoclass_content = "both"
autodoc_default_flags = ["members"]
autosummary_generate = True

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# Allow markdown includes (so releases.md can include CHANGLEOG.md)
# http://www.sphinx-doc.org/en/master/markdown.html
source_parsers = {".md": "recommonmark.parser.CommonMarkParser"}

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = [".rst", ".md"]

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = "index"

# General information about the project.
project = u"google-cloud-datacatalog"
copyright = u"2017, Google"
author = u"Google APIs"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The full version, including alpha/beta/rc tags.
release = __version__
# The short X.Y version.
version = ".".join(release.split(".")[0:2])

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["_build"]

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = "alabaster"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    "description": "Google Cloud Client Libraries for Python",
    "github_user": "googleapis",
    "github_repo": "google-cloud-python",
    "github_banner": True,
    "font_family": "'Roboto', Georgia, sans",
    "head_font_family": "'Roboto', Georgia, serif",
    "code_font_family": "'Roboto Mono', 'Consolas', monospace",
}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = []

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = "google-cloud-datacatalog-doc"

# -- Options for warnings ------------------------------------------------------

suppress_warnings = [
    # Temporarily suppress this to avoid "more than one target found for
    # cross-reference" warning, which are intractable for us to avoid while in
    # a mono-repo.
    # See https://github.com/sphinx-doc/sphinx/blob
    # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843
    "ref.python"
]

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',
    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',
    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
    # Latex figure (float) alignment
    #'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (
        master_doc,
        "google-cloud-datacatalog.tex",
        u"google-cloud-datacatalog Documentation",
        author,
        "manual",
    )
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (
        master_doc,
        "google-cloud-datacatalog",
        u"google-cloud-datacatalog Documentation",
        [author],
        1,
    )
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (
        master_doc,
        "google-cloud-datacatalog",
        u"google-cloud-datacatalog Documentation",
        author,
        "google-cloud-datacatalog",
        "GAPIC library for the {metadata.shortName} v1beta1 service",
        "APIs",
    )
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    "python": ("http://python.readthedocs.org/en/latest/", None),
    "gax": ("https://gax-python.readthedocs.org/en/latest/", None),
    "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
    "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None),
    "google.api_core": (
        "https://googleapis.github.io/google-cloud-python/latest",
        None,
    ),
    "grpc": ("https://grpc.io/grpc/python/", None),
    "requests": ("https://2.python-requests.org/en/master/", None),
    "fastavro": ("https://fastavro.readthedocs.io/en/stable/", None),
    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
}

# Napoleon settings
napoleon_google_docstring = True
napoleon_numpy_docstring = True
napoleon_include_private_with_doc = False
napoleon_include_special_with_doc = True
napoleon_use_admonition_for_examples = False
napoleon_use_admonition_for_notes = False
napoleon_use_admonition_for_references = False
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = True
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# lineage documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 19 23:32:11 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys

# http://docs.readthedocs.io/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules
autodoc_mock_imports = [
    "numpy",
    "pandas",
    "pandas.api.types",
    "matplotlib",
    "matplotlib.collections",
]

sys.path.insert(0, os.path.abspath("../"))

import lineage

# https://samnicholls.net/2016/06/15/how-to-sphinx-readthedocs/

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.viewcode",
    "sphinx.ext.githubpages",
    "sphinx.ext.napoleon",
]

napoleon_google_docstring = False
napoleon_use_param = False
napoleon_use_ivar = True

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"

# The master toctree document.
master_doc = "index"

# General information about the project.
project = "lineage"
copyright = "2017, Andrew Riha"
author = "Andrew Riha"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = lineage.__version__

# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    "**": [
        "about.html",
        "navigation.html",
        "relations.html",  # needs 'show_related': True theme option to display
        "searchbox.html",
        "donate.html",
    ]
}

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = "lineagedoc"

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',
    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',
    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',
    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, "lineage.tex", "lineage Documentation", "Andrew Riha", "manual")
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(master_doc, "lineage", "lineage Documentation", [author], 1)]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (
        master_doc,
        "lineage",
        "lineage Documentation",
        author,
        "lineage",
        "One line description of project.",
        "Miscellaneous",
    )
]
<commit_msg>Document `__init__` methods<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# lineage documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 19 23:32:11 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys

# http://docs.readthedocs.io/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules
autodoc_mock_imports = [
    "numpy",
    "pandas",
    "pandas.api.types",
    "matplotlib",
    "matplotlib.collections",
]

sys.path.insert(0, os.path.abspath("../"))

import lineage

# https://samnicholls.net/2016/06/15/how-to-sphinx-readthedocs/

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.viewcode",
    "sphinx.ext.githubpages",
    "sphinx.ext.napoleon",
]

napoleon_google_docstring = False
napoleon_use_param = False
napoleon_use_ivar = True

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"

# The master toctree document.
master_doc = "index"

# General information about the project.
project = "lineage"
copyright = "2017, Andrew Riha"
author = "Andrew Riha"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = lineage.__version__

# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    "**": [
        "about.html",
        "navigation.html",
        "relations.html",  # needs 'show_related': True theme option to display
        "searchbox.html",
        "donate.html",
    ]
}

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = "lineagedoc"

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',
    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',
    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',
    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, "lineage.tex", "lineage Documentation", "Andrew Riha", "manual")
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(master_doc, "lineage", "lineage Documentation", [author], 1)]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (
        master_doc,
        "lineage",
        "lineage Documentation",
        author,
        "lineage",
        "One line description of project.",
        "Miscellaneous",
    )
]

# document __init__ methods
# https://stackoverflow.com/a/5599712
def skip(app, what, name, obj, would_skip, options):
    if name == "__init__":
        return False
    return would_skip

def setup(app):
    app.connect("autodoc-skip-member", skip)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Apache Libcloud documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 31 12:16:27 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
              'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Apache Libcloud'
copyright = u'2013, The Apache Software Foundation'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.14.0'
# The full version, including alpha/beta/rc tags.
release = '0.14.0-dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = [
    '_build',
    '*/_*.rst'
]

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if on_rtd:
    html_theme = 'default'
    RTD_NEW_THEME = True
else:
    html_theme = 'nature'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static', '_static/images/']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'ApacheLibclouddoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'ApacheLibcloud.tex', u'Apache Libcloud Documentation',
   u'The Apache Software Foundation', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'apachelibcloud', u'Apache Libcloud Documentation',
     [u'The Apache Software Foundation'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'ApacheLibcloud', u'Apache Libcloud Documentation',
   u'The Apache Software Foundation', 'ApacheLibcloud', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

autoclass_content = 'both'
<commit_msg>Also generate api docs files on readthedocs.<commit_after># -*- coding: utf-8 -*-
#
# Apache Libcloud documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 31 12:16:27 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os
import subprocess

# Detect if we are running on read the docs
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if on_rtd:
    cmd = 'sphinx-apidoc -d 2 -o apidocs/ ../libcloud/'
    subprocess.call(cmd, shell=True)

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
              'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Apache Libcloud'
copyright = u'2013, The Apache Software Foundation'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.14.0'
# The full version, including alpha/beta/rc tags.
release = '0.14.0-dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = [
    '_build',
    '*/_*.rst'
]

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

if on_rtd:
    html_theme = 'default'
    RTD_NEW_THEME = True
else:
    html_theme = 'nature'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static', '_static/images/']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'ApacheLibclouddoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'ApacheLibcloud.tex', u'Apache Libcloud Documentation',
   u'The Apache Software Foundation', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'apachelibcloud', u'Apache Libcloud Documentation',
     [u'The Apache Software Foundation'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'ApacheLibcloud', u'Apache Libcloud Documentation',
   u'The Apache Software Foundation', 'ApacheLibcloud', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

autoclass_content = 'both'
<|endoftext|>


<commit_before>##header begins
from collections import defaultdict
from extract_keywords import split_line
import numpy
import csv

##header ends
def author_topic(total_data, num_topics=150):
	assert('Author' in total_data.keys())
	assert('PaperAuthor' in total_data.keys())
	assert('Paper' in total_data.keys())
	assert('title_keyword' in total_data.keys())

	paper = total_data['Paper']
	author = total_data['Author']
	paper_author = total_data['PaperAuthor']
	author_sum = dict()

	for (paper_id,author_id) in paper_author.keys():
		if (not paper_id in paper.keys()):
			sum_vector = numpy.zeros(num_topics)
		else:
			sum_vector = paper[paper_id]['topic_sum']
		if not (author_id in author_sum.keys()):
			author_sum[author_id] = sum_vector
		else:
			author_sum[author_id] = author_sum[author_id] + sum_vector

	for (author_id, author_field) in author.items():
		if author_id in author_sum.keys():
			vector = author_sum[author_id]
			sum = numpy.sum(vector)
			if(sum > 0.5):
				vector /= sum
			author_field['topic_sum'] = vector
		else:
			author_field['topic_sum'] = numpy.zeros(num_topics)

def paper_author_topic_sum(total_data, num_topics=150):
	assert('Author' in total_data.keys())
	assert('PaperAuthor' in total_data.keys())
	assert('Paper' in total_data.keys())
	assert('title_keyword' in total_data.keys())

	paper = total_data['Paper']
	author = total_data['Author']
	paper_author = total_data['PaperAuthor']

	for (paper_id,author_id) in paper_author.keys():
		if (paper_id in paper.keys()) and (author_id in author.keys()):
			paper_topic = paper[paper_id]['topic_sum']
			author_topic = author[author_id]['topic_sum']
			norm = numpy.linalg.norm(paper_topic)
			if norm > 0.5:
				paper_author[(paper_id, author_id)]['topic_dot'] = numpy.dot(paper_topic, author_topic)
				paper_author[(paper_id, author_id)]['has_topic_dot'] = int(1)
			else:
				paper_author[(paper_id, author_id)]['topic_dot'] = float(0)
				paper_author[(paper_id, author_id)]['has_topic_dot'] = int(0)
		else:
			paper_author[(paper_id, author_id)]['topic_dot'] = float(0)
			paper_author[(paper_id, author_id)]['has_topic_dot'] = int(0)

def save_topic_sum(total_data):
	assert('PaperAuthor' in total_data.keys())
	paper_author = total_data['PaperAuthor']
	with open('paper_author_topic_dot.csv','w',encoding='utf-8') as write_file:
		writer = csv.writer(write_file)
		writer.writerow(['paperid','authorid','topic_dot','has_topic_dot'])
		for (paper_id,author_id) in paper_author.keys():
			writer.writerow([paper_id, author_id,
			                 paper_author[(paper_id,author_id)]['topic_dot'],
			                 paper_author[(paper_id,author_id)]['has_topic_dot']])

##from here, leeopop does
def merge_authors(total_data):
	assert('Author' in total_data.keys())
	author_data = total_data['Author']
	name_data = defaultdict(set)

	for (id, fields) in author_data.items():
		name_data[fields['name']].add(id)

	old_to_new = dict()
	new_to_old = dict()
	for (new_id, same_keys) in zip(range(len(name_data)), name_data.values()):
		for old in same_keys:
			old_to_new[old] = new_id
		new_to_old[new_id] = same_keys

	total_data['old_to_new'] = old_to_new
	total_data['new_to_old'] = new_to_old

def paper_topic(total_data):
	assert('title_keyword' in total_data.keys())
	assert('title_keyword_topic' in total_data.keys())
	paper = total_data['Paper']
	# key: word, value: unique id
	# e.g. title_keyword['apple']['unique'] returns unique identifier
	title_keyword = total_data['title_keyword']
	# key: word id (unique identifier), value: numpy vector
	title_keyword_topic = total_data['title_keyword_topic']
	dim = len(list(title_keyword_topic.values())[0])
	for item in paper:
		title_vector = split_line(paper[item]['title'])
		title_vector += split_line(paper[item]['keyword'])
		topic_vector = numpy.zeros(dim)
		for word in title_vector:
			if word in title_keyword:
				word_id = int(title_keyword[word]['unique'])
				word_vector = title_keyword_topic[word_id]
				topic_vector += word_vector
		sum = numpy.sum(topic_vector)
		if sum > 0.5:
			topic_vector /= sum
		paper[item]['topic_sum'] = topic_vector
	#hint: use split_line in extract_keyword.py
	#hint: merge 'title' and 'keyword' of paper data

##end

##from here, gangok does

##end<commit_msg>add cross vector<commit_after>##header begins
from collections import defaultdict
from extract_keywords import split_line
import numpy
import csv
__do_cross = True
##header ends
def author_topic(total_data, num_topics=150):
	assert('Author' in total_data.keys())
	assert('PaperAuthor' in total_data.keys())
	assert('Paper' in total_data.keys())
	assert('title_keyword' in total_data.keys())

	paper = total_data['Paper']
	author = total_data['Author']
	paper_author = total_data['PaperAuthor']
	author_sum = dict()

	for (paper_id,author_id) in paper_author.keys():
		if (not paper_id in paper.keys()):
			sum_vector = numpy.zeros(num_topics)
		else:
			sum_vector = paper[paper_id]['topic_sum']
		if not (author_id in author_sum.keys()):
			author_sum[author_id] = sum_vector
		else:
			author_sum[author_id] = author_sum[author_id] + sum_vector

	for (author_id, author_field) in author.items():
		if author_id in author_sum.keys():
			vector = author_sum[author_id]
			sum = numpy.sum(vector)
			if(sum > 0.5):
				vector /= sum
			author_field['topic_sum'] = vector
		else:
			author_field['topic_sum'] = numpy.zeros(num_topics)

def paper_author_topic_sum(total_data, num_topics=150):
	assert('Author' in total_data.keys())
	assert('PaperAuthor' in total_data.keys())
	assert('Paper' in total_data.keys())
	assert('title_keyword' in total_data.keys())

	paper = total_data['Paper']
	author = total_data['Author']
	paper_author = total_data['PaperAuthor']

	for (paper_id,author_id) in paper_author.keys():
		if (paper_id in paper.keys()) and (author_id in author.keys()):
			paper_topic = paper[paper_id]['topic_sum']
			author_topic = author[author_id]['topic_sum']
			sum = numpy.sum(paper_topic)
			if sum > 0.5:
				paper_author[(paper_id, author_id)]['topic_dot'] = numpy.dot(paper_topic, author_topic)
				paper_author[(paper_id, author_id)]['has_topic_dot'] = int(1)
				if __do_cross:
					paper_author[(paper_id, author_id)]['topic_cross'] = numpy.multiply(paper_topic, author_topic)
			else:
				paper_author[(paper_id, author_id)]['topic_dot'] = float(0)
				paper_author[(paper_id, author_id)]['has_topic_dot'] = int(0)
				if __do_cross:
					paper_author[(paper_id, author_id)]['topic_cross'] = numpy.zeros(num_topics)
		else:
			paper_author[(paper_id, author_id)]['topic_dot'] = float(0)
			paper_author[(paper_id, author_id)]['has_topic_dot'] = int(0)
			if __do_cross:
				paper_author[(paper_id, author_id)]['topic_cross'] = numpy.zeros(num_topics)

def save_topic_sum(total_data, num_topics=150):
	assert('PaperAuthor' in total_data.keys())
	paper_author = total_data['PaperAuthor']
	with open('paper_author_topic_dot.csv','w',encoding='utf-8') as write_file:
		writer = csv.writer(write_file)
		col = ['paperid','authorid','topic_dot','has_topic_dot']
		if __do_cross:
			for i in range(num_topics):
				col.append('topic_cross_{}'.format(i))
		writer.writerow(col)
		for (paper_id,author_id) in paper_author.keys():
			row = [paper_id, author_id,
			                 paper_author[(paper_id,author_id)]['topic_dot'],
			                 paper_author[(paper_id,author_id)]['has_topic_dot']]
			if __do_cross:
				cross_vector = paper_author[(paper_id,author_id)]['topic_cross']
				for i in range(num_topics):
					row.append(cross_vector[i])
			writer.writerow(row)

##from here, leeopop does
def merge_authors(total_data):
	assert('Author' in total_data.keys())
	author_data = total_data['Author']
	name_data = defaultdict(set)

	for (id, fields) in author_data.items():
		name_data[fields['name']].add(id)

	old_to_new = dict()
	new_to_old = dict()
	for (new_id, same_keys) in zip(range(len(name_data)), name_data.values()):
		for old in same_keys:
			old_to_new[old] = new_id
		new_to_old[new_id] = same_keys

	total_data['old_to_new'] = old_to_new
	total_data['new_to_old'] = new_to_old

def paper_topic(total_data):
	assert('title_keyword' in total_data.keys())
	assert('title_keyword_topic' in total_data.keys())
	paper = total_data['Paper']
	# key: word, value: unique id
	# e.g. title_keyword['apple']['unique'] returns unique identifier
	title_keyword = total_data['title_keyword']
	# key: word id (unique identifier), value: numpy vector
	title_keyword_topic = total_data['title_keyword_topic']
	dim = len(list(title_keyword_topic.values())[0])
	for item in paper:
		title_vector = split_line(paper[item]['title'])
		title_vector += split_line(paper[item]['keyword'])
		topic_vector = numpy.zeros(dim)
		for word in title_vector:
			if word in title_keyword:
				word_id = int(title_keyword[word]['unique'])
				word_vector = title_keyword_topic[word_id]
				topic_vector += word_vector
		sum = numpy.sum(topic_vector)
		if sum > 0.5:
			topic_vector /= sum
		paper[item]['topic_sum'] = topic_vector
	#hint: use split_line in extract_keyword.py
	#hint: merge 'title' and 'keyword' of paper data

##end

##from here, gangok does

##end<|endoftext|>