<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds perf trybots that can run telemetry tests."""

import json
import logging
import os
import re
import subprocess
import sys
import urllib2

from telemetry.core import platform
from telemetry.core.platform import trybot_device
from telemetry.core import possible_browser
from telemetry import decorators

CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg'
BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg'
SUCCESS, NO_CHANGES, ERROR = range(3)
# Unsupported Perf bisect bots.
EXCLUDED_BOTS = {
    'win_xp_perf_bisect',
    'linux_perf_tester',
    'linux_perf_bisector',
    'win_perf_bisect_builder',
    'win_x64_perf_bisect_builder',
    'linux_perf_bisect_builder',
    'mac_perf_bisect_builder',
    'android_perf_bisect_builder'
}

INCLUDE_BOTS = [
    'trybot-all',
    'trybot-all-win',
    'trybot-all-mac',
    'trybot-all-linux',
    'trybot-all-android'
]

class TrybotError(Exception):

  def __str__(self):
    return '%s\nError running tryjob.' % self.args[0]

class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
  """A script that sends a job to a trybot."""

  def __init__(self, browser_type, _):
    target_os = browser_type.split('-')[1]
    self._builder_names = _GetBuilderNames(browser_type)
    super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True)

  def Create(self, finder_options):
    raise NotImplementedError()

  def SupportsOptions(self, finder_options):
    if ((finder_options.device and finder_options.device != 'trybot') or
        finder_options.chrome_root or
        finder_options.cros_remote or
        finder_options.extensions_to_load or
        finder_options.profile_dir):
      return False
    return True

  def IsRemote(self):
    return True

  def _RunProcess(self, cmd):
    logging.debug('Running process: "%s"', ' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    returncode = proc.poll()
    return (returncode, out, err)

  def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path):
    """Updates perf config file, uploads changes and excutes perf try job.

    Args:
      bot_platform: Name of the platform to be generated.
      cfg_file_path: Perf config file path.

    Returns:
      (result, msg) where result is one of:
          SUCCESS if a tryjob was sent
          NO_CHANGES if there was nothing to try,
          ERROR if a tryjob was attempted but an error encountered
          and msg is an error message if an error was encountered, or rietveld
          url if success, otherwise throws TrybotError exception.
    """
    config = self._GetPerfConfig(bot_platform)
    try:
      config_file = open(cfg_file_path, 'w')
    except IOError:
      msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
      return (ERROR, msg)
    config_file.write('config = %s' % json.dumps(
        config, sort_keys=True, indent=2, separators=(',', ': ')))
    config_file.close()
    # Commit the config changes locally.
    returncode, out, err = self._RunProcess(
        ['git', 'commit', '-a', '-m', 'bisect config: %s' % bot_platform])
    if returncode:
      raise TrybotError('Could not commit bisect config change for %s,'
                        ' error %s' % (bot_platform, err))
    # Upload the CL to rietveld and run a try job.
    returncode, out, err = self._RunProcess([
        'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
        'CL for perf tryjob on %s' % bot_platform
    ])
    if returncode:
      raise TrybotError('Could upload to rietveld for %s, error %s' %
                        (bot_platform, err))

    match = re.search(r'https://codereview.chromium.org/[\d]+', out)
    if not match:
      raise TrybotError('Could not upload CL to rietveld for %s! Output %s' %
                        (bot_platform, out))
    rietveld_url = match.group(0)
    # Generate git try command for available bots.
    git_try_command = ['git', 'cl', 'try', '-m', 'tryserver.chromium.perf']
    for bot in self._builder_names[bot_platform]:
      git_try_command.extend(['-b', bot])
    returncode, out, err = self._RunProcess(git_try_command)
    if returncode:
      raise TrybotError('Could not try CL for %s, error %s' %
                        (bot_platform, err))

    return (SUCCESS, rietveld_url)

  def _GetPerfConfig(self, bot_platform):
    """Generates the perf config for try job.

    Args:
      bot_platform: Name of the platform to be generated.

    Returns:
      A dictionary with perf config parameters.
    """
    # Generate the command line for the perf trybots
    target_arch = 'ia32'
    arguments = sys.argv
    if bot_platform in ['win', 'win-x64']:
      arguments[0] = 'python tools\\perf\\run_benchmark'
    else:
      arguments[0] = './tools/perf/run_benchmark'
    for index, arg in enumerate(arguments):
      if arg.startswith('--browser='):
        if bot_platform == 'android':
          arguments[index] = '--browser=android-chrome-shell'
        elif any('x64' in bot for bot in self._builder_names[bot_platform]):
          arguments[index] = '--browser=release_x64'
          target_arch = 'x64'
        else:
          arguments[index] = '--browser=release'
    command = ' '.join(arguments)

    return {
        'command': command,
        'repeat_count': '1',
        'max_time_minutes': '120',
        'truncate_percent': '0',
        'target_arch': target_arch,
    }

  def _AttemptTryjob(self, cfg_file_path):
    """Attempts to run a tryjob from the current directory.

    This is run once for chromium, and if it returns NO_CHANGES, once for blink.

    Args:
      cfg_file_path: Path to the config file for the try job.

    Returns:
      Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to
      try, ERROR if a tryjob was attempted but an error encountered.
    """
    source_repo = 'chromium'
    if cfg_file_path == BLINK_CONFIG_FILENAME:
      source_repo = 'blink'

    # TODO(prasadv): This method is quite long, we should consider refactor
    # this by extracting to helper methods.
    returncode, original_branchname, err = self._RunProcess(
        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    if returncode:
      msg = 'Must be in a git repository to send changes to trybots.'
      if err:
        msg += '\nGit error: %s' % err
      logging.error(msg)
      return ERROR

    original_branchname = original_branchname.strip()

    # Check if the tree is dirty: make sure the index is up to date and then
    # run diff-index
    self._RunProcess(['git', 'update-index', '--refresh', '-q'])
    returncode, out, err = self._RunProcess(['git', 'diff-index', 'HEAD'])
    if out:
      logging.error(
          'Cannot send a try job with a dirty tree. Commit locally first.')
      return ERROR

    # Make sure the tree does have local commits.
    returncode, out, err = self._RunProcess(
        ['git', 'log', 'origin/master..HEAD'])
    if not out:
      return NO_CHANGES

    # Create/check out the telemetry-tryjob branch, and edit the configs
    # for the tryjob there.
    returncode, out, err = self._RunProcess(
        ['git', 'checkout', '-b', 'telemetry-tryjob'])
    if returncode:
      logging.error('Error creating branch telemetry-tryjob. '
                    'Please delete it if it exists.\n%s', err)
      return ERROR
    try:
      returncode, out, err = self._RunProcess(
          ['git', 'branch', '--set-upstream-to', 'origin/master'])
      if returncode:
        logging.error('Error in git branch --set-upstream-to: %s', err)
        return ERROR
      for bot_platform in self._builder_names:
        try:
          results, output = self._UpdateConfigAndRunTryjob(
              bot_platform, cfg_file_path)
          if results == ERROR:
            logging.error(output)
            return ERROR
          print ('Uploaded %s try job to rietveld for %s platform. '
                 'View progress at %s' % (source_repo, bot_platform, output))
        except TrybotError, err:
          print err
          logging.error(err)
    finally:
      # Checkout original branch and delete telemetry-tryjob branch.
      # TODO(prasadv): This finally block could be extracted out to be a
      # separate function called _CleanupBranch.
      returncode, out, err = self._RunProcess(
          ['git', 'checkout', original_branchname])
      if returncode:
        logging.error('Could not check out %s. Please check it out and '
                      'manually delete the telemetry-tryjob branch. '
                      ': %s', original_branchname, err)
        return ERROR # pylint: disable=lost-exception
      logging.info('Checked out original branch: %s', original_branchname)
      returncode, out, err = self._RunProcess(
          ['git', 'branch', '-D', 'telemetry-tryjob'])
      if returncode:
        logging.error('Could not delete telemetry-tryjob branch. '
                      'Please delete it manually: %s', err)
        return ERROR # pylint: disable=lost-exception
      logging.info('Deleted temp branch: telemetry-tryjob')
    return SUCCESS

  def RunRemote(self):
    """Sends a tryjob to a perf trybot.

    This creates a branch, telemetry-tryjob, switches to that branch, edits
    the bisect config, commits it, uploads the CL to rietveld, and runs a
    tryjob on the given bot.
    """
    # First check if there are chromium changes to upload.
    status = self._AttemptTryjob(CHROMIUM_CONFIG_FILENAME)
    if status not in [SUCCESS, ERROR]:
      # If we got here, there are no chromium changes to upload. Try blink.
      os.chdir('third_party/WebKit/')
      status = self._AttemptTryjob(BLINK_CONFIG_FILENAME)
      os.chdir('../..')
      if status not in [SUCCESS, ERROR]:
        logging.error('No local changes found in chromium or blink trees. '
                      'browser=%s argument sends local changes to the '
                      'perf trybot(s): %s.', self.browser_type,
                      self._builder_names.values())

  def _InitPlatformIfNeeded(self):
    if self._platform:
      return

    self._platform = platform.GetHostPlatform()

    # pylint: disable=W0212
    self._platform_backend = self._platform._platform_backend

def SelectDefaultBrowser(_):
  return None

def CanFindAvailableBrowsers():
  return True

@decorators.Cache
def _GetTrybotList():
  f = urllib2.urlopen(
      'http://build.chromium.org/p/tryserver.chromium.perf/json')
  builders = json.loads(f.read()).get('builders', {}).keys()
  builders = ['trybot-%s' % bot.replace('_perf_bisect', '').replace('_', '-')
              for bot in builders if bot not in EXCLUDED_BOTS]
  builders.extend(INCLUDE_BOTS)
  return sorted(builders)

def _GetBuilderNames(browser_type):
  """ Return platform and its available bot name as dictionary."""
  if 'all' not in browser_type:
    bot = ['%s_perf_bisect' % browser_type.replace(
        'trybot-', '').replace('-', '_')]
    bot_platform = browser_type.split('-')[1]
    if 'x64' in browser_type:
      bot_platform += '-x64'
    return {bot_platform: bot}

  f = urllib2.urlopen(
      'http://build.chromium.org/p/tryserver.chromium.perf/json')
  builders = json.loads(f.read()).get('builders', {}).keys()
  # Exclude unsupported bots like win xp and some dummy bots.
  builders = [bot for bot in builders if bot not in EXCLUDED_BOTS]

  platform_and_bots = {}
  for os_name in ['linux', 'android', 'mac', 'win']:
    platform_and_bots[os_name] = [bot for bot in builders if os_name in bot]

  # Special case for Windows x64, consider it as separate platform
  # config config should contain target_arch=x64 and --browser=release_x64.
  win_x64_bots = [platform_and_bots['win'].pop(i)
      for i, win_bot in enumerate(platform_and_bots['win']) if 'x64' in win_bot]
  platform_and_bots['win-x64'] = win_x64_bots

  if 'all-win' in browser_type:
    return {'win': platform_and_bots['win'],
            'win-x64': platform_and_bots['win-x64']}
  if 'all-mac' in browser_type:
    return {'mac': platform_and_bots['mac']}
  if 'all-android' in browser_type:
    return {'android': platform_and_bots['android']}
  if 'all-linux' in browser_type:
    return {'linux': platform_and_bots['linux']}

  return platform_and_bots

def FindAllBrowserTypes(finder_options):
  # Listing browsers requires an http request; only do this if the user is
  # running with browser=list or a browser=trybot-* argument.
  if (finder_options.browser_type and
      (finder_options.browser_type == 'list' or
       finder_options.browser_type.startswith('trybot'))):
    return _GetTrybotList()
  return []

def FindAllAvailableBrowsers(finder_options, device):
  """Find all perf trybots on tryserver.chromium.perf."""
  if not isinstance(device, trybot_device.TrybotDevice):
    return []

  return [PossibleTrybotBrowser(b, finder_options) for b in
          FindAllBrowserTypes(finder_options)]
<commit_msg>Add android arm64 bisect builder to the list of excluded bisect bots for perf trybot.<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds perf trybots that can run telemetry tests."""

import json
import logging
import os
import re
import subprocess
import sys
import urllib2

from telemetry.core import platform
from telemetry.core.platform import trybot_device
from telemetry.core import possible_browser
from telemetry import decorators

CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg'
BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg'
SUCCESS, NO_CHANGES, ERROR = range(3)
# Unsupported Perf bisect bots.
EXCLUDED_BOTS = {
    'win_xp_perf_bisect',
    'linux_perf_tester',
    'linux_perf_bisector',
    'win_perf_bisect_builder',
    'win_x64_perf_bisect_builder',
    'linux_perf_bisect_builder',
    'mac_perf_bisect_builder',
    'android_perf_bisect_builder',
    'android_arm64_perf_bisect_builder'
}

INCLUDE_BOTS = [
    'trybot-all',
    'trybot-all-win',
    'trybot-all-mac',
    'trybot-all-linux',
    'trybot-all-android'
]

class TrybotError(Exception):

  def __str__(self):
    return '%s\nError running tryjob.' % self.args[0]

class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
  """A script that sends a job to a trybot."""

  def __init__(self, browser_type, _):
    target_os = browser_type.split('-')[1]
    self._builder_names = _GetBuilderNames(browser_type)
    super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True)

  def Create(self, finder_options):
    raise NotImplementedError()

  def SupportsOptions(self, finder_options):
    if ((finder_options.device and finder_options.device != 'trybot') or
        finder_options.chrome_root or
        finder_options.cros_remote or
        finder_options.extensions_to_load or
        finder_options.profile_dir):
      return False
    return True

  def IsRemote(self):
    return True

  def _RunProcess(self, cmd):
    logging.debug('Running process: "%s"', ' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    returncode = proc.poll()
    return (returncode, out, err)

  def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path):
    """Updates perf config file, uploads changes and excutes perf try job.

    Args:
      bot_platform: Name of the platform to be generated.
      cfg_file_path: Perf config file path.

    Returns:
      (result, msg) where result is one of:
          SUCCESS if a tryjob was sent
          NO_CHANGES if there was nothing to try,
          ERROR if a tryjob was attempted but an error encountered
          and msg is an error message if an error was encountered, or rietveld
          url if success, otherwise throws TrybotError exception.
    """
    config = self._GetPerfConfig(bot_platform)
    try:
      config_file = open(cfg_file_path, 'w')
    except IOError:
      msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
      return (ERROR, msg)
    config_file.write('config = %s' % json.dumps(
        config, sort_keys=True, indent=2, separators=(',', ': ')))
    config_file.close()
    # Commit the config changes locally.
    returncode, out, err = self._RunProcess(
        ['git', 'commit', '-a', '-m', 'bisect config: %s' % bot_platform])
    if returncode:
      raise TrybotError('Could not commit bisect config change for %s,'
                        ' error %s' % (bot_platform, err))
    # Upload the CL to rietveld and run a try job.
    returncode, out, err = self._RunProcess([
        'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
        'CL for perf tryjob on %s' % bot_platform
    ])
    if returncode:
      raise TrybotError('Could upload to rietveld for %s, error %s' %
                        (bot_platform, err))

    match = re.search(r'https://codereview.chromium.org/[\d]+', out)
    if not match:
      raise TrybotError('Could not upload CL to rietveld for %s! Output %s' %
                        (bot_platform, out))
    rietveld_url = match.group(0)
    # Generate git try command for available bots.
    git_try_command = ['git', 'cl', 'try', '-m', 'tryserver.chromium.perf']
    for bot in self._builder_names[bot_platform]:
      git_try_command.extend(['-b', bot])
    returncode, out, err = self._RunProcess(git_try_command)
    if returncode:
      raise TrybotError('Could not try CL for %s, error %s' %
                        (bot_platform, err))

    return (SUCCESS, rietveld_url)

  def _GetPerfConfig(self, bot_platform):
    """Generates the perf config for try job.

    Args:
      bot_platform: Name of the platform to be generated.

    Returns:
      A dictionary with perf config parameters.
    """
    # Generate the command line for the perf trybots
    target_arch = 'ia32'
    arguments = sys.argv
    if bot_platform in ['win', 'win-x64']:
      arguments[0] = 'python tools\\perf\\run_benchmark'
    else:
      arguments[0] = './tools/perf/run_benchmark'
    for index, arg in enumerate(arguments):
      if arg.startswith('--browser='):
        if bot_platform == 'android':
          arguments[index] = '--browser=android-chrome-shell'
        elif any('x64' in bot for bot in self._builder_names[bot_platform]):
          arguments[index] = '--browser=release_x64'
          target_arch = 'x64'
        else:
          arguments[index] = '--browser=release'
    command = ' '.join(arguments)

    return {
        'command': command,
        'repeat_count': '1',
        'max_time_minutes': '120',
        'truncate_percent': '0',
        'target_arch': target_arch,
    }

  def _AttemptTryjob(self, cfg_file_path):
    """Attempts to run a tryjob from the current directory.

    This is run once for chromium, and if it returns NO_CHANGES, once for blink.

    Args:
      cfg_file_path: Path to the config file for the try job.

    Returns:
      Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to
      try, ERROR if a tryjob was attempted but an error encountered.
    """
    source_repo = 'chromium'
    if cfg_file_path == BLINK_CONFIG_FILENAME:
      source_repo = 'blink'

    # TODO(prasadv): This method is quite long, we should consider refactor
    # this by extracting to helper methods.
    returncode, original_branchname, err = self._RunProcess(
        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    if returncode:
      msg = 'Must be in a git repository to send changes to trybots.'
      if err:
        msg += '\nGit error: %s' % err
      logging.error(msg)
      return ERROR

    original_branchname = original_branchname.strip()

    # Check if the tree is dirty: make sure the index is up to date and then
    # run diff-index
    self._RunProcess(['git', 'update-index', '--refresh', '-q'])
    returncode, out, err = self._RunProcess(['git', 'diff-index', 'HEAD'])
    if out:
      logging.error(
          'Cannot send a try job with a dirty tree. Commit locally first.')
      return ERROR

    # Make sure the tree does have local commits.
    returncode, out, err = self._RunProcess(
        ['git', 'log', 'origin/master..HEAD'])
    if not out:
      return NO_CHANGES

    # Create/check out the telemetry-tryjob branch, and edit the configs
    # for the tryjob there.
    returncode, out, err = self._RunProcess(
        ['git', 'checkout', '-b', 'telemetry-tryjob'])
    if returncode:
      logging.error('Error creating branch telemetry-tryjob. '
                    'Please delete it if it exists.\n%s', err)
      return ERROR
    try:
      returncode, out, err = self._RunProcess(
          ['git', 'branch', '--set-upstream-to', 'origin/master'])
      if returncode:
        logging.error('Error in git branch --set-upstream-to: %s', err)
        return ERROR
      for bot_platform in self._builder_names:
        try:
          results, output = self._UpdateConfigAndRunTryjob(
              bot_platform, cfg_file_path)
          if results == ERROR:
            logging.error(output)
            return ERROR
          print ('Uploaded %s try job to rietveld for %s platform. '
                 'View progress at %s' % (source_repo, bot_platform, output))
        except TrybotError, err:
          print err
          logging.error(err)
    finally:
      # Checkout original branch and delete telemetry-tryjob branch.
      # TODO(prasadv): This finally block could be extracted out to be a
      # separate function called _CleanupBranch.
      returncode, out, err = self._RunProcess(
          ['git', 'checkout', original_branchname])
      if returncode:
        logging.error('Could not check out %s. Please check it out and '
                      'manually delete the telemetry-tryjob branch. '
                      ': %s', original_branchname, err)
        return ERROR # pylint: disable=lost-exception
      logging.info('Checked out original branch: %s', original_branchname)
      returncode, out, err = self._RunProcess(
          ['git', 'branch', '-D', 'telemetry-tryjob'])
      if returncode:
        logging.error('Could not delete telemetry-tryjob branch. '
                      'Please delete it manually: %s', err)
        return ERROR # pylint: disable=lost-exception
      logging.info('Deleted temp branch: telemetry-tryjob')
    return SUCCESS

  def RunRemote(self):
    """Sends a tryjob to a perf trybot.

    This creates a branch, telemetry-tryjob, switches to that branch, edits
    the bisect config, commits it, uploads the CL to rietveld, and runs a
    tryjob on the given bot.
    """
    # First check if there are chromium changes to upload.
    status = self._AttemptTryjob(CHROMIUM_CONFIG_FILENAME)
    if status not in [SUCCESS, ERROR]:
      # If we got here, there are no chromium changes to upload. Try blink.
      os.chdir('third_party/WebKit/')
      status = self._AttemptTryjob(BLINK_CONFIG_FILENAME)
      os.chdir('../..')
      if status not in [SUCCESS, ERROR]:
        logging.error('No local changes found in chromium or blink trees. '
                      'browser=%s argument sends local changes to the '
                      'perf trybot(s): %s.', self.browser_type,
                      self._builder_names.values())

  def _InitPlatformIfNeeded(self):
    if self._platform:
      return

    self._platform = platform.GetHostPlatform()

    # pylint: disable=W0212
    self._platform_backend = self._platform._platform_backend

def SelectDefaultBrowser(_):
  return None

def CanFindAvailableBrowsers():
  return True

@decorators.Cache
def _GetTrybotList():
  f = urllib2.urlopen(
      'http://build.chromium.org/p/tryserver.chromium.perf/json')
  builders = json.loads(f.read()).get('builders', {}).keys()
  builders = ['trybot-%s' % bot.replace('_perf_bisect', '').replace('_', '-')
              for bot in builders if bot not in EXCLUDED_BOTS]
  builders.extend(INCLUDE_BOTS)
  return sorted(builders)

def _GetBuilderNames(browser_type):
  """ Return platform and its available bot name as dictionary."""
  if 'all' not in browser_type:
    bot = ['%s_perf_bisect' % browser_type.replace(
        'trybot-', '').replace('-', '_')]
    bot_platform = browser_type.split('-')[1]
    if 'x64' in browser_type:
      bot_platform += '-x64'
    return {bot_platform: bot}

  f = urllib2.urlopen(
      'http://build.chromium.org/p/tryserver.chromium.perf/json')
  builders = json.loads(f.read()).get('builders', {}).keys()
  # Exclude unsupported bots like win xp and some dummy bots.
  builders = [bot for bot in builders if bot not in EXCLUDED_BOTS]

  platform_and_bots = {}
  for os_name in ['linux', 'android', 'mac', 'win']:
    platform_and_bots[os_name] = [bot for bot in builders if os_name in bot]

  # Special case for Windows x64, consider it as separate platform
  # config config should contain target_arch=x64 and --browser=release_x64.
  win_x64_bots = [platform_and_bots['win'].pop(i)
      for i, win_bot in enumerate(platform_and_bots['win']) if 'x64' in win_bot]
  platform_and_bots['win-x64'] = win_x64_bots

  if 'all-win' in browser_type:
    return {'win': platform_and_bots['win'],
            'win-x64': platform_and_bots['win-x64']}
  if 'all-mac' in browser_type:
    return {'mac': platform_and_bots['mac']}
  if 'all-android' in browser_type:
    return {'android': platform_and_bots['android']}
  if 'all-linux' in browser_type:
    return {'linux': platform_and_bots['linux']}

  return platform_and_bots

def FindAllBrowserTypes(finder_options):
  # Listing browsers requires an http request; only do this if the user is
  # running with browser=list or a browser=trybot-* argument.
  if (finder_options.browser_type and
      (finder_options.browser_type == 'list' or
       finder_options.browser_type.startswith('trybot'))):
    return _GetTrybotList()
  return []

def FindAllAvailableBrowsers(finder_options, device):
  """Find all perf trybots on tryserver.chromium.perf."""
  if not isinstance(device, trybot_device.TrybotDevice):
    return []

  return [PossibleTrybotBrowser(b, finder_options) for b in
          FindAllBrowserTypes(finder_options)]
<|endoftext|>


<commit_before>import hmac
import hashlib

class Utility(object):
    def __init__(self, client=None):
        self.client = client

    def verify_payment_signature(self, parameters):
        order_id = parameters['order_id']
        payment_id = parameters['payment_id']
        razorpay_signature = parameters['razorpay_signature']
        msg = "{}|{}".format(order_id, payment_id)
        dig = hmac.new(key=self.client.auth[1],
                       msg=msg,
                       digestmod=hashlib.sha256)

        generated_signature = dig.hexdigest()

        if not hmac.compare_digest(generated_signature, razorpay_signature):
            raise ValueError('Payment Signature Verification Failed')

        return True
<commit_msg>[python-sdk] encode to str<commit_after>import hmac
import hashlib
import sys

class Utility(object):
    def __init__(self, client=None):
        self.client = client

    def verify_payment_signature(self, parameters):
        order_id = parameters['order_id']
        payment_id = parameters['payment_id']
        razorpay_signature = parameters['razorpay_signature']
        msg = "{}|{}".format(order_id, payment_id)
        key = self.client.auth[1]

        if sys.version_info[0] == 3:
            key = bytes(key, 'utf-8')
            msg = bytes(msg, 'utf-8')

        dig = hmac.new(key=key,
                       msg=msg,
                       digestmod=hashlib.sha256)

        generated_signature = dig.hexdigest()

        if not hmac.compare_digest(generated_signature, razorpay_signature):
            raise ValueError('Payment Signature Verification Failed')

        return True
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================
"""Experimental support for defining XLA shardings."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as _np  # Avoids becoming a part of public Tensorflow API.

from tensorflow.compiler.tf2xla.python import xla as tf2xla
from tensorflow.compiler.xla import xla_data_pb2
from tensorflow.core.framework import attr_value_pb2

class Sharding(object):
  """A class to support adding sharding attributes to Ops.

  Use the factory constructors and then call apply_to_tensor:
    Sharding.replicate().apply_to_tensor(tensor)
  """

  def __init__(self, proto=None):
    """Do not use this constructor; use the factory functions below."""
    self._proto = proto

  @classmethod
  def replicate(cls):
    """Returns a replicated sharding attribute.

    This causes an op to be computed in its entirety independently on all
    cores in the XLA device.
    """
    return Sharding(
        proto=xla_data_pb2.OpSharding(type=xla_data_pb2.OpSharding.REPLICATED))

  @classmethod
  def assign_device(cls, core):
    """Returns an AssignDevice sharding attribute.

    This causes an op to be computed in its entirety only on one core in
    the XLA device.
    Args:
      core: The core to assign this Op to.
    """
    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.MAXIMAL,
            tile_assignment_dimensions=[1],
            tile_assignment_devices=[core]))

  @classmethod
  def tile(cls, tile_assignment):
    """Returns a Tiled sharding attribute.

    This causes an op to be partially computed on multiple cores in the
    XLA device.

    Args:
      tile_assignment: An np.ndarray describing the topology of the tiling and
        which device will compute which part of the topology.

    Raises:
      TypeError: tile_assignment was not of np.array type.

    TODO(jmolloy): This concept is nefarious and is not
    something we really want to expose to users (especially as the
    contract for tile_assignment is very strict).
    """
    if not isinstance(tile_assignment, _np.ndarray):
      raise TypeError('Tile assignment must be of type np.ndarray')
    dims = list(tile_assignment.shape)
    flattened_devices = tile_assignment.reshape(-1, order='C')
    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.OTHER,
            tile_assignment_dimensions=dims,
            tile_assignment_devices=list(flattened_devices)))

  @classmethod
  def split(cls, tensor, split_dimension, num_devices):
    """Returns a Sharding that splits a tensor across a dimension.

    This creates a Tiled attribute, similar to tile(), but easier to use for the
    common case of tiling a tensor N ways in one dimension.

    Args:
      tensor: A tf.Tensor to split.
      split_dimension: The dimension number to split.
      num_devices: The number of cores to split `tensor` over.

    Raises:
      ValueError: The tensor to split was smaller in the split dimension than
        the number of devices to split over.
    """
    shape = tensor.shape.as_list()
    if (shape[split_dimension] is not None and
        shape[split_dimension] < num_devices):
      raise ValueError('Split dimension was smaller than the required number '
                       'of splits: shape=%r, dimension=%r, num_devices=%r' %
                       (shape, split_dimension, num_devices))

    tile_assignment_dims = [1] * len(shape)
    tile_assignment_dims[split_dimension] = num_devices

    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.OTHER,
            tile_assignment_dimensions=tile_assignment_dims,
            tile_assignment_devices=range(num_devices)))

  def apply_to_tensor(self, tensor, assign_tuple_sharding=False):
    """Applies this Sharding attribute to `tensor`.

    Args:
      tensor: A tf.Tensor to split.
      assign_tuple_sharding: If the sharding type should be a tuple.
    """
    if len(tensor.op.outputs) > 1 or assign_tuple_sharding:
      proto = self._get_or_create_tuple_proto(tensor.op)
      # We can't mutate an element of old_proto.tuple_shardings, so create
      # a new proto.
      tuple_shardings = list(proto.tuple_shardings)
      tuple_shardings[tensor.value_index] = self._proto
      proto = xla_data_pb2.OpSharding(
          type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=tuple_shardings)
    else:
      proto = self._proto
    attr_value = attr_value_pb2.AttrValue(s=proto.SerializeToString())
    # TODO(jmolloy): This need to be seriously revisited before declaring this
    # API available for public use.
    # pylint: disable=protected-access
    tensor.op._set_attr('_XlaSharding', attr_value)

  @property
  def proto(self):
    """Return the sharding protobuf of type xla_data_pb2.OpSharding."""
    return self._proto

  def _get_or_create_tuple_proto(self, op):
    try:
      attr = op.get_attr('_XlaSharding')
      proto = xla_data_pb2.OpSharding()
      proto.ParseFromString(attr)
      return proto
    except ValueError:
      return self._create_tuple_proto(op)

  def _create_tuple_proto(self, op):
    shardings = [
        xla_data_pb2.OpSharding(type=xla_data_pb2.OpSharding.REPLICATED)
        for _ in op.outputs
    ]
    return xla_data_pb2.OpSharding(
        type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=shardings)

# Helpers for the above factory functions that allow easy application of
# shardings, for example:
#   tensor = xla_sharding.replicate(tensor)

def replicate(tensor, assign_tuple_sharding=False, use_sharding_op=False):
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.replicate().apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor

def assign_device(tensor, device, assign_tuple_sharding=False):
  Sharding.assign_device(device).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor

def tile(tensor,
         tile_assignment,
         assign_tuple_sharding=False,
         use_sharding_op=False):
  """Returns a tensor that has tiled sharding.

  Args:
    tensor: A tf.Tensor to shard.
    tile_assignment: An np.ndarray describing the topology of the tiling and
      which device will compute which part of the topology.
    assign_tuple_sharding: If the sharding type should be a tuple.
    use_sharding_op: If true, adds a sharding op to set the sharding.
  """
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.tile(tile_assignment).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding
  )
  return tensor

def split(tensor,
          split_dimension,
          num_devices,
          assign_tuple_sharding=False,
          use_sharding_op=False):
  """Returns a tensor that is split along the given dimension.

  Args:
    tensor: A tf.Tensor to split.
    split_dimension: The dimension to split.
    num_devices: The number of devices to partition the dimension.
    assign_tuple_sharding: If the sharding type should be a tuple.
    use_sharding_op: If true, adds a sharding op to set the sharding.
  """
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.split(tensor, split_dimension, num_devices).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor
<commit_msg>Add parameter to use sharding op for xla_sharding.assign_device() API.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================
"""Experimental support for defining XLA shardings."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as _np  # Avoids becoming a part of public Tensorflow API.

from tensorflow.compiler.tf2xla.python import xla as tf2xla
from tensorflow.compiler.xla import xla_data_pb2
from tensorflow.core.framework import attr_value_pb2

class Sharding(object):
  """A class to support adding sharding attributes to Ops.

  Use the factory constructors and then call apply_to_tensor:
    Sharding.replicate().apply_to_tensor(tensor)
  """

  def __init__(self, proto=None):
    """Do not use this constructor; use the factory functions below."""
    self._proto = proto

  @classmethod
  def replicate(cls):
    """Returns a replicated sharding attribute.

    This causes an op to be computed in its entirety independently on all
    cores in the XLA device.
    """
    return Sharding(
        proto=xla_data_pb2.OpSharding(type=xla_data_pb2.OpSharding.REPLICATED))

  @classmethod
  def assign_device(cls, core):
    """Returns an AssignDevice sharding attribute.

    This causes an op to be computed in its entirety only on one core in
    the XLA device.
    Args:
      core: The core to assign this Op to.
    """
    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.MAXIMAL,
            tile_assignment_dimensions=[1],
            tile_assignment_devices=[core]))

  @classmethod
  def tile(cls, tile_assignment):
    """Returns a Tiled sharding attribute.

    This causes an op to be partially computed on multiple cores in the
    XLA device.

    Args:
      tile_assignment: An np.ndarray describing the topology of the tiling and
        which device will compute which part of the topology.

    Raises:
      TypeError: tile_assignment was not of np.array type.

    TODO(jmolloy): This concept is nefarious and is not
    something we really want to expose to users (especially as the
    contract for tile_assignment is very strict).
    """
    if not isinstance(tile_assignment, _np.ndarray):
      raise TypeError('Tile assignment must be of type np.ndarray')
    dims = list(tile_assignment.shape)
    flattened_devices = tile_assignment.reshape(-1, order='C')
    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.OTHER,
            tile_assignment_dimensions=dims,
            tile_assignment_devices=list(flattened_devices)))

  @classmethod
  def split(cls, tensor, split_dimension, num_devices):
    """Returns a Sharding that splits a tensor across a dimension.

    This creates a Tiled attribute, similar to tile(), but easier to use for the
    common case of tiling a tensor N ways in one dimension.

    Args:
      tensor: A tf.Tensor to split.
      split_dimension: The dimension number to split.
      num_devices: The number of cores to split `tensor` over.

    Raises:
      ValueError: The tensor to split was smaller in the split dimension than
        the number of devices to split over.
    """
    shape = tensor.shape.as_list()
    if (shape[split_dimension] is not None and
        shape[split_dimension] < num_devices):
      raise ValueError('Split dimension was smaller than the required number '
                       'of splits: shape=%r, dimension=%r, num_devices=%r' %
                       (shape, split_dimension, num_devices))

    tile_assignment_dims = [1] * len(shape)
    tile_assignment_dims[split_dimension] = num_devices

    return Sharding(
        proto=xla_data_pb2.OpSharding(
            type=xla_data_pb2.OpSharding.OTHER,
            tile_assignment_dimensions=tile_assignment_dims,
            tile_assignment_devices=range(num_devices)))

  def apply_to_tensor(self, tensor, assign_tuple_sharding=False):
    """Applies this Sharding attribute to `tensor`.

    Args:
      tensor: A tf.Tensor to split.
      assign_tuple_sharding: If the sharding type should be a tuple.
    """
    if len(tensor.op.outputs) > 1 or assign_tuple_sharding:
      proto = self._get_or_create_tuple_proto(tensor.op)
      # We can't mutate an element of old_proto.tuple_shardings, so create
      # a new proto.
      tuple_shardings = list(proto.tuple_shardings)
      tuple_shardings[tensor.value_index] = self._proto
      proto = xla_data_pb2.OpSharding(
          type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=tuple_shardings)
    else:
      proto = self._proto
    attr_value = attr_value_pb2.AttrValue(s=proto.SerializeToString())
    # TODO(jmolloy): This need to be seriously revisited before declaring this
    # API available for public use.
    # pylint: disable=protected-access
    tensor.op._set_attr('_XlaSharding', attr_value)

  @property
  def proto(self):
    """Return the sharding protobuf of type xla_data_pb2.OpSharding."""
    return self._proto

  def _get_or_create_tuple_proto(self, op):
    try:
      attr = op.get_attr('_XlaSharding')
      proto = xla_data_pb2.OpSharding()
      proto.ParseFromString(attr)
      return proto
    except ValueError:
      return self._create_tuple_proto(op)

  def _create_tuple_proto(self, op):
    shardings = [
        xla_data_pb2.OpSharding(type=xla_data_pb2.OpSharding.REPLICATED)
        for _ in op.outputs
    ]
    return xla_data_pb2.OpSharding(
        type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=shardings)

# Helpers for the above factory functions that allow easy application of
# shardings, for example:
#   tensor = xla_sharding.replicate(tensor)

def replicate(tensor, assign_tuple_sharding=False, use_sharding_op=False):
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.replicate().apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor

def assign_device(tensor,
                  device,
                  assign_tuple_sharding=False,
                  use_sharding_op=False):
  """Returns a tensor that has AssignDevice sharding attribute."""
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)

  Sharding.assign_device(device).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor

def tile(tensor,
         tile_assignment,
         assign_tuple_sharding=False,
         use_sharding_op=False):
  """Returns a tensor that has tiled sharding.

  Args:
    tensor: A tf.Tensor to shard.
    tile_assignment: An np.ndarray describing the topology of the tiling and
      which device will compute which part of the topology.
    assign_tuple_sharding: If the sharding type should be a tuple.
    use_sharding_op: If true, adds a sharding op to set the sharding.
  """
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.tile(tile_assignment).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding
  )
  return tensor

def split(tensor,
          split_dimension,
          num_devices,
          assign_tuple_sharding=False,
          use_sharding_op=False):
  """Returns a tensor that is split along the given dimension.

  Args:
    tensor: A tf.Tensor to split.
    split_dimension: The dimension to split.
    num_devices: The number of devices to partition the dimension.
    assign_tuple_sharding: If the sharding type should be a tuple.
    use_sharding_op: If true, adds a sharding op to set the sharding.
  """
  if use_sharding_op:
    tensor = tf2xla.sharding(tensor)
  Sharding.split(tensor, split_dimension, num_devices).apply_to_tensor(
      tensor,
      assign_tuple_sharding=assign_tuple_sharding)
  return tensor
<|endoftext|>


<commit_before># pylint: disable=g-bad-file-header
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A tf.learn implementation of tensor_forest (extremely random forests)."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time

import numpy as np
import six

from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib.learn.python.learn import monitors as mon

from tensorflow.contrib.learn.python.learn.estimators import estimator
from tensorflow.contrib.learn.python.learn.estimators import run_config

from tensorflow.contrib.tensor_forest.client import eval_metrics
from tensorflow.contrib.tensor_forest.data import data_ops
from tensorflow.contrib.tensor_forest.python import tensor_forest

from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops

class LossMonitor(mon.EveryN):
  """Terminates training when training loss stops decreasing."""

  def __init__(self,
               early_stopping_rounds,
               every_n_steps):
    super(LossMonitor, self).__init__(every_n_steps=every_n_steps)
    self.early_stopping_rounds = early_stopping_rounds
    self.min_loss = None
    self.min_loss_step = 0

  def set_estimator(self, est):
    """This function gets called in the same graph as _get_train_ops."""
    super(LossMonitor, self).set_estimator(est)
    self._loss_op_name = est.training_loss.name

  def every_n_step_end(self, step, outputs):
    super(LossMonitor, self).every_n_step_end(step, outputs)
    current_loss = outputs[self._loss_op_name]
    if self.min_loss is None or current_loss < self.min_loss:
      self.min_loss = current_loss
      self.min_loss_step = step
    return step - self.min_loss_step >= self.early_stopping_rounds

class TensorForestEstimator(estimator.BaseEstimator):
  """An estimator that can train and evaluate a random forest."""

  def __init__(self, params, device_assigner=None, model_dir=None,
               graph_builder_class=tensor_forest.RandomForestGraphs,
               master='', accuracy_metric=None,
               tf_random_seed=None, continue_training=False, verbose=1,
               max_to_keep=5, save_checkpoint_secs=300):
    self.params = params.fill()
    self.accuracy_metric = (accuracy_metric or
                            ('r2' if self.params.regression else 'accuracy'))
    self.data_feeder = None
    self.device_assigner = (
        device_assigner or tensor_forest.RandomForestDeviceAssigner())
    self.graph_builder_class = graph_builder_class
    self.training_args = {}
    self.construction_args = {}

    config = run_config.RunConfig(
        master=master,
        tf_random_seed=(tf_random_seed or int((time.time() * 1000) % 1000)),
        save_checkpoints_secs=save_checkpoint_secs,
        keep_checkpoint_max=max_to_keep)

    super(TensorForestEstimator, self).__init__(model_dir=model_dir,
                                                config=config)

  def predict_proba(self, x=None, input_fn=None, batch_size=None):
    """Returns prediction probabilities for given features (classification).

    Args:
      x: features.
      input_fn: Input function. If set, x and y must be None.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted probabilities.

    Raises:
      ValueError: If both or neither of x and input_fn were given.
    """
    return super(TensorForestEstimator, self).predict(
        x=x, input_fn=input_fn, batch_size=batch_size)

  def predict(self, x=None, input_fn=None, axis=None, batch_size=None):
    """Returns predictions for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x must be None.
      axis: Axis on which to argmax (for classification).
            Last axis is used by default.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted classes or regression values.
    """
    probabilities = self.predict_proba(x, input_fn, batch_size)
    if self.params.regression:
      return probabilities
    else:
      return np.argmax(probabilities, axis=1)

  def _get_train_ops(self, features, targets):
    """Method that builds model graph and returns trainer ops.

    Args:
      features: `Tensor` or `dict` of `Tensor` objects.
      targets: `Tensor` or `dict` of `Tensor` objects.

    Returns:
      Tuple of train `Operation` and loss `Tensor`.
    """
    features, spec = data_ops.ParseDataTensorOrDict(features)
    labels = data_ops.ParseLabelTensorOrDict(targets)

    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner,
        **self.construction_args)

    epoch = None
    if self.data_feeder:
      epoch = self.data_feeder.make_epoch_variable()

    train = control_flow_ops.group(
        graph_builder.training_graph(
            features, labels, data_spec=spec, epoch=epoch,
            **self.training_args),
        state_ops.assign_add(contrib_framework.get_global_step(), 1))

    self.training_loss = graph_builder.training_loss()

    return train, self.training_loss

  def _get_predict_ops(self, features):
    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner, training=False,
        **self.construction_args)
    features, spec = data_ops.ParseDataTensorOrDict(features)
    return graph_builder.inference_graph(features, data_spec=spec)

  def _get_eval_ops(self, features, targets, metrics):
    features, spec = data_ops.ParseDataTensorOrDict(features)
    labels = data_ops.ParseLabelTensorOrDict(targets)

    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner, training=False,
        **self.construction_args)

    probabilities = graph_builder.inference_graph(features, data_spec=spec)

    # One-hot the labels.
    if not self.params.regression:
      labels = math_ops.to_int64(array_ops.one_hot(math_ops.to_int64(
          array_ops.squeeze(labels)), self.params.num_classes, 1, 0))

    if metrics is None:
      metrics = {self.accuracy_metric:
                 eval_metrics.get_metric(self.accuracy_metric)}

    result = {}
    for name, metric in six.iteritems(metrics):
      result[name] = metric(probabilities, labels)

    return result
<commit_msg>TensorForestEstimator - removed unused continue_training and changed to use config in constructor (#3244)<commit_after># pylint: disable=g-bad-file-header
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A tf.learn implementation of tensor_forest (extremely random forests)."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time

import numpy as np
import six

from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib.learn.python.learn import monitors as mon

from tensorflow.contrib.learn.python.learn.estimators import estimator
from tensorflow.contrib.learn.python.learn.estimators import run_config

from tensorflow.contrib.tensor_forest.client import eval_metrics
from tensorflow.contrib.tensor_forest.data import data_ops
from tensorflow.contrib.tensor_forest.python import tensor_forest

from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops

class LossMonitor(mon.EveryN):
  """Terminates training when training loss stops decreasing."""

  def __init__(self,
               early_stopping_rounds,
               every_n_steps):
    super(LossMonitor, self).__init__(every_n_steps=every_n_steps)
    self.early_stopping_rounds = early_stopping_rounds
    self.min_loss = None
    self.min_loss_step = 0

  def set_estimator(self, est):
    """This function gets called in the same graph as _get_train_ops."""
    super(LossMonitor, self).set_estimator(est)
    self._loss_op_name = est.training_loss.name

  def every_n_step_end(self, step, outputs):
    super(LossMonitor, self).every_n_step_end(step, outputs)
    current_loss = outputs[self._loss_op_name]
    if self.min_loss is None or current_loss < self.min_loss:
      self.min_loss = current_loss
      self.min_loss_step = step
    return step - self.min_loss_step >= self.early_stopping_rounds

class TensorForestEstimator(estimator.BaseEstimator):
  """An estimator that can train and evaluate a random forest."""

  def __init__(self, params, device_assigner=None, model_dir=None,
               graph_builder_class=tensor_forest.RandomForestGraphs,
               master='', accuracy_metric=None,
               tf_random_seed=None, config=None):
    self.params = params.fill()
    self.accuracy_metric = (accuracy_metric or
                            ('r2' if self.params.regression else 'accuracy'))
    self.data_feeder = None
    self.device_assigner = (
        device_assigner or tensor_forest.RandomForestDeviceAssigner())
    self.graph_builder_class = graph_builder_class
    self.training_args = {}
    self.construction_args = {}

    super(TensorForestEstimator, self).__init__(model_dir=model_dir,
                                                config=config)

  def predict_proba(self, x=None, input_fn=None, batch_size=None):
    """Returns prediction probabilities for given features (classification).

    Args:
      x: features.
      input_fn: Input function. If set, x and y must be None.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted probabilities.

    Raises:
      ValueError: If both or neither of x and input_fn were given.
    """
    return super(TensorForestEstimator, self).predict(
        x=x, input_fn=input_fn, batch_size=batch_size)

  def predict(self, x=None, input_fn=None, axis=None, batch_size=None):
    """Returns predictions for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x must be None.
      axis: Axis on which to argmax (for classification).
            Last axis is used by default.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted classes or regression values.
    """
    probabilities = self.predict_proba(x, input_fn, batch_size)
    if self.params.regression:
      return probabilities
    else:
      return np.argmax(probabilities, axis=1)

  def _get_train_ops(self, features, targets):
    """Method that builds model graph and returns trainer ops.

    Args:
      features: `Tensor` or `dict` of `Tensor` objects.
      targets: `Tensor` or `dict` of `Tensor` objects.

    Returns:
      Tuple of train `Operation` and loss `Tensor`.
    """
    features, spec = data_ops.ParseDataTensorOrDict(features)
    labels = data_ops.ParseLabelTensorOrDict(targets)

    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner,
        **self.construction_args)

    epoch = None
    if self.data_feeder:
      epoch = self.data_feeder.make_epoch_variable()

    train = control_flow_ops.group(
        graph_builder.training_graph(
            features, labels, data_spec=spec, epoch=epoch,
            **self.training_args),
        state_ops.assign_add(contrib_framework.get_global_step(), 1))

    self.training_loss = graph_builder.training_loss()

    return train, self.training_loss

  def _get_predict_ops(self, features):
    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner, training=False,
        **self.construction_args)
    features, spec = data_ops.ParseDataTensorOrDict(features)
    return graph_builder.inference_graph(features, data_spec=spec)

  def _get_eval_ops(self, features, targets, metrics):
    features, spec = data_ops.ParseDataTensorOrDict(features)
    labels = data_ops.ParseLabelTensorOrDict(targets)

    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner, training=False,
        **self.construction_args)

    probabilities = graph_builder.inference_graph(features, data_spec=spec)

    # One-hot the labels.
    if not self.params.regression:
      labels = math_ops.to_int64(array_ops.one_hot(math_ops.to_int64(
          array_ops.squeeze(labels)), self.params.num_classes, 1, 0))

    if metrics is None:
      metrics = {self.accuracy_metric:
                 eval_metrics.get_metric(self.accuracy_metric)}

    result = {}
    for name, metric in six.iteritems(metrics):
      result[name] = metric(probabilities, labels)

    return result
<|endoftext|>


<commit_before>"""Factory for creating devices."""
import logging
import math
import time

from google.appengine.api import namespace_manager
from google.appengine.ext import ndb

import flask

from appengine import model, pushrpc, rest
from common import detector

DEVICE_TYPES = {}

def static_command(func):
  """Device command decorator - automatically dispatches methods."""
  setattr(func, 'is_command', True)
  setattr(func, 'is_static', True)
  return func

def register(device_type):
  """Decorator to cause device types to be registered."""
  def class_rebuilder(cls):
    DEVICE_TYPES[device_type] = cls
    return cls
  return class_rebuilder

def create_device(device_id, body, device_type=None):
  """Factory for creating new devices."""
  if device_type is None:
    device_type = body.pop('type', None)

  if device_type is None:
    flask.abort(400, '\'type\' field expected in body.')

  constructor = DEVICE_TYPES.get(device_type, None)
  if constructor is None:
    logging.error('No device type \'%s\'', device_type)
    flask.abort(400)
  return constructor(id=device_id)

class Device(model.Base):
  """Base class for all device drivers."""

  # This is the name the user sets
  name = ndb.StringProperty(required=False)

  # This is the (optional) name read from the device itself
  device_name = ndb.StringProperty(required=False)

  last_update = ndb.DateTimeProperty(required=False, auto_now=True)
  room = ndb.StringProperty()

  # What can I do with this device? ie SWITCH, DIMMABLE, COLOR_TEMP etc
  capabilities = ndb.ComputedProperty(lambda self: self.get_capabilities(),
                                      repeated=True)

  # What broad category does this device belong to?  LIGHTING, CLIMATE, MUSIC
  categories = ndb.ComputedProperty(lambda self: self.get_categories(),
                                    repeated=True)

  def get_capabilities(self):
    return []

  def get_categories(self):
    return []

  @classmethod
  def _event_classname(cls):
    return 'device'

  def handle_event(self, event):
    pass

  @classmethod
  def handle_static_event(cls, event):
    pass

  @classmethod
  def get_by_capability(cls, capability):
    return cls.query(Device.capabilities == capability)

  def find_room(self):
    """Resolve the room for this device.  May return null."""
    if not self.room:
      return None

    # This is a horrible hack, but room imports devices,
    # so need to be lazy here.
    from appengine import room
    room_obj = room.Room.get_by_id(self._device.room)
    if not room_obj:
      return None

    return room_obj

class DetectorMixin(object):
  """Add a failure detector to a device to interpret motion sensor data."""
  detector = ndb.JsonProperty()

  # These fields represent the real state of this sensor
  # and when the real state last changes
  occupied = ndb.BooleanProperty(default=False)
  occupied_last_update = ndb.IntegerProperty()

  # These fields represent the inferred state of this sensor
  # after processing by the failure detector, and the time
  # when this inferred state last changed.
  inferred_state = ndb.BooleanProperty(default=False)
  inferred_last_update = ndb.IntegerProperty(default=0)

  #def to_dict(self):
  #  """We don't need to expose the detector in the dict repr."""
  #  # Mainly as its too big for pusher
  #  result = super(DetectorMixin, self).to_dict()
  #  del result['detector']
  #  return result

  def _load_detector(self):
    """Either return the a rehydrated detector, or a fresh one."""
    if self.detector is None:
      return detector.AccrualFailureDetector()
    else:
      return detector.AccrualFailureDetector.from_dict(self.detector)

  def is_occupied(self):
    """Use a failure detector to determine state of sensor"""
    # Does the sensor say we're occupied?
    if self.occupied:
      return True

    # If not, does the detector?
    instance = self._load_detector()
    inferred_state = instance.is_alive()

    if self.inferred_state != inferred_state:
      self.inferred_state = inferred_state
      self.inferred_last_update = int(time.time())
      self.put()

    return inferred_state

  def real_occupied_state_change(self, state):
    """The underly state changed; synthensize events and save the detector."""
    # I'm getting dupe events; ignore until
    # I figure out why.
    if state == self.state:
      return

    instance = self._load_detector()
    now = int(time.time())

    # Construct an appropriate set of fake heartbeats and
    # feed them to the detector.
    # As we don't get heart beats from the motion sensors,
    # we just fake them.  We only do this if the sensor
    # is transitions from occupied -> not occupied.  We
    # don't need to do it for the other way as we just use
    # the real state.
    if self.occupied and self.occupied_last_update is not None:
      # We know the first hit was at self.occupied_last_update.
      # We know the sensor can't have recieved a hit in the past
      # timeout seconds, so the last hit was now - timeout ago
      # Otherwise, we're going to put a bunch of hits inbetween
      # those times
      timeout = 240
      start = self.occupied_last_update
      end = now - timeout
      assert start < end

      diff = end - start
      count = math.ceil(diff * 1.0 / timeout)
      inc = diff / count
      for i in xrange(int(count)):
        instance.heartbeat(start + int(i * inc))

    # save the detector and other fields
    # no need to put this object, plumbing in device.py
    # will do that for us.
    self.detector = instance.to_dict()
    self.occupied = state
    self.occupied_last_update = now

    room = self.find_room()
    if room:
      room.update_lights()

class Switch(Device):
  """A switch."""
  # Represents the actual state of the switch; changing this
  # (and calling update()) will changed the switch.
  state = ndb.BooleanProperty(default=False)

  # Represents the state the user wants, and when they asked for
  # it.  Most of the time users will control rooms etc, not individual
  # lights.  But its possible.
  # UI should set this and call update_lights on the room.
  intended_state = ndb.BooleanProperty()
  state_last_update = ndb.IntegerProperty(default=0)

  def get_capabilities(self):
    return ['SWITCH']

  def get_categories(self):
    return ['LIGHTING']

# pylint: disable=invalid-name
blueprint = flask.Blueprint('device', __name__)
rest.register_class(blueprint, Device, create_device)

def process_events(events):
  """Process a set of events."""

  device_cache = {}

  for event in events:
    device_type = event['device_type']
    device_id = event['device_id']
    event_body = event['event']

    if device_id is None:
      DEVICE_TYPES[device_type].handle_static_event(event_body)
      continue

    if device_id in device_cache:
      device = device_cache[device_id]
    else:
      device = Device.get_by_id(device_id)
      if not device:
        device = create_device(device_id, None,
                               device_type=device_type)
      device_cache[device_id] = device

    device.handle_event(event_body)

  ndb.put_multi(device_cache.values())

@blueprint.route('/events', methods=['POST'])
def handle_events():
  """Handle events from devices."""

  # This endpoint needs to authenticate itself.
  proxy = pushrpc.authenticate()
  if proxy is None:
    flask.abort(401)

  # If proxy hasn't been claimed, not much we can do.
  if proxy.building_id is None:
    logging.info('Dropping events as this proxy is not claimed')
    return ('', 204)

  # We need to set namespace - not done by main.py
  namespace_manager.set_namespace(proxy.building_id)

  events = flask.request.get_json()
  logging.info('Processing %d events', len(events))

  process_events(events)

  return ('', 204)

<commit_msg>Don't die when we get in a bad state, log it and carry on<commit_after>"""Factory for creating devices."""
import logging
import math
import time

from google.appengine.api import namespace_manager
from google.appengine.ext import ndb

import flask

from appengine import model, pushrpc, rest
from common import detector

DEVICE_TYPES = {}

def static_command(func):
  """Device command decorator - automatically dispatches methods."""
  setattr(func, 'is_command', True)
  setattr(func, 'is_static', True)
  return func

def register(device_type):
  """Decorator to cause device types to be registered."""
  def class_rebuilder(cls):
    DEVICE_TYPES[device_type] = cls
    return cls
  return class_rebuilder

def create_device(device_id, body, device_type=None):
  """Factory for creating new devices."""
  if device_type is None:
    device_type = body.pop('type', None)

  if device_type is None:
    flask.abort(400, '\'type\' field expected in body.')

  constructor = DEVICE_TYPES.get(device_type, None)
  if constructor is None:
    logging.error('No device type \'%s\'', device_type)
    flask.abort(400)
  return constructor(id=device_id)

class Device(model.Base):
  """Base class for all device drivers."""

  # This is the name the user sets
  name = ndb.StringProperty(required=False)

  # This is the (optional) name read from the device itself
  device_name = ndb.StringProperty(required=False)

  last_update = ndb.DateTimeProperty(required=False, auto_now=True)
  room = ndb.StringProperty()

  # What can I do with this device? ie SWITCH, DIMMABLE, COLOR_TEMP etc
  capabilities = ndb.ComputedProperty(lambda self: self.get_capabilities(),
                                      repeated=True)

  # What broad category does this device belong to?  LIGHTING, CLIMATE, MUSIC
  categories = ndb.ComputedProperty(lambda self: self.get_categories(),
                                    repeated=True)

  def get_capabilities(self):
    return []

  def get_categories(self):
    return []

  @classmethod
  def _event_classname(cls):
    return 'device'

  def handle_event(self, event):
    pass

  @classmethod
  def handle_static_event(cls, event):
    pass

  @classmethod
  def get_by_capability(cls, capability):
    return cls.query(Device.capabilities == capability)

  def find_room(self):
    """Resolve the room for this device.  May return null."""
    if not self.room:
      return None

    # This is a horrible hack, but room imports devices,
    # so need to be lazy here.
    from appengine import room
    room_obj = room.Room.get_by_id(self._device.room)
    if not room_obj:
      return None

    return room_obj

class DetectorMixin(object):
  """Add a failure detector to a device to interpret motion sensor data."""
  detector = ndb.JsonProperty()

  # These fields represent the real state of this sensor
  # and when the real state last changes
  occupied = ndb.BooleanProperty(default=False)
  occupied_last_update = ndb.IntegerProperty()

  # These fields represent the inferred state of this sensor
  # after processing by the failure detector, and the time
  # when this inferred state last changed.
  inferred_state = ndb.BooleanProperty(default=False)
  inferred_last_update = ndb.IntegerProperty(default=0)

  #def to_dict(self):
  #  """We don't need to expose the detector in the dict repr."""
  #  # Mainly as its too big for pusher
  #  result = super(DetectorMixin, self).to_dict()
  #  del result['detector']
  #  return result

  def _load_detector(self):
    """Either return the a rehydrated detector, or a fresh one."""
    if self.detector is None:
      return detector.AccrualFailureDetector()
    else:
      return detector.AccrualFailureDetector.from_dict(self.detector)

  def is_occupied(self):
    """Use a failure detector to determine state of sensor"""
    # Does the sensor say we're occupied?
    if self.occupied:
      return True

    # If not, does the detector?
    instance = self._load_detector()
    inferred_state = instance.is_alive()

    if self.inferred_state != inferred_state:
      self.inferred_state = inferred_state
      self.inferred_last_update = int(time.time())
      self.put()

    return inferred_state

  def real_occupied_state_change(self, state):
    """The underly state changed; synthensize events and save the detector."""
    # I'm getting dupe events; ignore until
    # I figure out why.
    if state == self.state:
      return

    instance = self._load_detector()
    now = int(time.time())

    # Construct an appropriate set of fake heartbeats and
    # feed them to the detector.
    # As we don't get heart beats from the motion sensors,
    # we just fake them.  We only do this if the sensor
    # is transitions from occupied -> not occupied.  We
    # don't need to do it for the other way as we just use
    # the real state.
    if self.occupied and self.occupied_last_update is not None:
      # We know the first hit was at self.occupied_last_update.
      # We know the sensor can't have recieved a hit in the past
      # timeout seconds, so the last hit was now - timeout ago
      # Otherwise, we're going to put a bunch of hits inbetween
      # those times
      timeout = 240
      start = self.occupied_last_update
      end = now - timeout
      if start > end:
        logging.error("This shouldn't happen; start = %s < end = %s",
                      start, end)
        start = end - 1

      diff = end - start
      count = math.ceil(diff * 1.0 / timeout)
      inc = diff / count
      for i in xrange(int(count)):
        instance.heartbeat(start + int(i * inc))

    # save the detector and other fields
    # no need to put this object, plumbing in device.py
    # will do that for us.
    self.detector = instance.to_dict()
    self.occupied = state
    self.occupied_last_update = now

    room = self.find_room()
    if room:
      room.update_lights()

class Switch(Device):
  """A switch."""
  # Represents the actual state of the switch; changing this
  # (and calling update()) will changed the switch.
  state = ndb.BooleanProperty(default=False)

  # Represents the state the user wants, and when they asked for
  # it.  Most of the time users will control rooms etc, not individual
  # lights.  But its possible.
  # UI should set this and call update_lights on the room.
  intended_state = ndb.BooleanProperty()
  state_last_update = ndb.IntegerProperty(default=0)

  def get_capabilities(self):
    return ['SWITCH']

  def get_categories(self):
    return ['LIGHTING']

# pylint: disable=invalid-name
blueprint = flask.Blueprint('device', __name__)
rest.register_class(blueprint, Device, create_device)

def process_events(events):
  """Process a set of events."""

  device_cache = {}

  for event in events:
    device_type = event['device_type']
    device_id = event['device_id']
    event_body = event['event']

    if device_id is None:
      DEVICE_TYPES[device_type].handle_static_event(event_body)
      continue

    if device_id in device_cache:
      device = device_cache[device_id]
    else:
      device = Device.get_by_id(device_id)
      if not device:
        device = create_device(device_id, None,
                               device_type=device_type)
      device_cache[device_id] = device

    device.handle_event(event_body)

  ndb.put_multi(device_cache.values())

@blueprint.route('/events', methods=['POST'])
def handle_events():
  """Handle events from devices."""

  # This endpoint needs to authenticate itself.
  proxy = pushrpc.authenticate()
  if proxy is None:
    flask.abort(401)

  # If proxy hasn't been claimed, not much we can do.
  if proxy.building_id is None:
    logging.info('Dropping events as this proxy is not claimed')
    return ('', 204)

  # We need to set namespace - not done by main.py
  namespace_manager.set_namespace(proxy.building_id)

  events = flask.request.get_json()
  logging.info('Processing %d events', len(events))

  process_events(events)

  return ('', 204)

<|endoftext|>


<commit_before># import os
# from google.appengine.dist import use_library
# os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
# use_library('django', '1.2')

appstats_CALC_RPC_COSTS = True
# webapp_django_version = '1.2'

def webapp_add_wsgi_middleware(app):
    "Add the appstats middleware"
    from google.appengine.ext.appstats import recording
    app = recording.appstats_wsgi_middleware(app)
    return app
<commit_msg>Changed Django version<commit_after>import os
from google.appengine.dist import use_library
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
use_library('django', '1.2')

appstats_CALC_RPC_COSTS = True
webapp_django_version = '1.2'

def webapp_add_wsgi_middleware(app):
    "Add the appstats middleware"
    from google.appengine.ext.appstats import recording
    app = recording.appstats_wsgi_middleware(app)
    return app
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Low-level functions if you want to build your own higher level abstractions.

.. warning::
    This is a "Hazardous Materials" module.  You should **ONLY** use it if
    you’re 100% absolutely sure that you know what you’re doing because this
    module is full of land mines, dragons, and dinosaurs with laser guns.
"""

from __future__ import absolute_import, division, print_function

from enum import Enum

from six import PY3

from ._ffi import ffi, lib
from ._util import _get_encoded_len
from .exceptions import VerificationError, HashingError

__all__ = [
    "Type",
    "ffi",
    "hash_secret",
    "hash_secret_raw",
    "verify_secret",
]

class Type(Enum):
    """
    Enum of Argon2 variants.
    """
    D = lib.Argon2_d
    """
    Argon2\ **d** is faster and uses data-depending memory access, which makes
    it less suitable for hashing secrets and more suitable for cryptocurrencies
    and applications with no threats from side-channel timing attacks.
    """
    I = lib.Argon2_i
    """
    Argon2\ **i** uses data-independent memory access, which is preferred for
    password hashing and password-based key derivation.  Argon2i is slower as
    it makes more passes over the memory to protect from tradeoff attacks.
    """

def hash_secret(secret, salt, time_cost, memory_cost, parallelism, hash_len,
                type):
    """
    Hash *secret* and return an **encoded** hash.

    An encoded hash can be directly passed into :func:`verify_secret` as it
    contains all parameters and the salt.

    :param bytes secret: Secret to hash.
    :param bytes salt: A salt_.  Should be random and different for each
        secret.
    :param Type type: Which Argon2 variant to use.

    For an explanation of the Argon2 parameters see :class:`PasswordHasher`.

    :rtype: bytes

    :raises argon2.exceptions.HashingError: If hashing fails.

    .. versionadded:: 16.0.0

    .. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography)
    .. _kibibytes: https://en.wikipedia.org/wiki/Binary_prefix#kibi
    """
    size = _get_encoded_len(hash_len, len(salt))
    buf = ffi.new("uint8_t[]", size)
    rv = lib.argon2_hash(
        time_cost, memory_cost, parallelism,
        ffi.new("uint8_t[]", secret), len(secret),
        ffi.new("uint8_t[]", salt), len(salt),
        ffi.NULL, hash_len,
        buf, size,
        type.value,
    )
    if rv != lib.ARGON2_OK:
        raise HashingError(error_to_str(rv))

    return ffi.string(buf)

def hash_secret_raw(secret, salt, time_cost, memory_cost, parallelism,
                    hash_len, type):
    """
    Hash *password* and return a **raw** hash.

    This function takes the same parameters as :func:`hash_secret`.
    """
    buf = ffi.new("uint8_t[]", hash_len)

    rv = lib.argon2_hash(
        time_cost, memory_cost, parallelism,
        ffi.new("uint8_t[]", secret), len(secret),
        ffi.new("uint8_t[]", salt), len(salt),
        buf, hash_len,
        ffi.NULL, 0,
        type.value,
    )
    if rv != lib.ARGON2_OK:
        raise HashingError(error_to_str(rv))

    return bytes(ffi.buffer(buf, hash_len))

def verify_secret(hash, secret, type):
    """
    Verify whether *secret* is correct for *hash* of *type*.

    :param bytes hash: An encoded Argon2 hash as returned by
        :func:`hash_secret`.
    :param bytes secret: The secret to verify whether it matches the one
        in *hash*.
    :param Type type: Type for *hash*.

    :raises argon2.exceptions.VerificationError: If verification fails.

    :return: ``True`` on success, raise
            :exc:`~argon2.exceptions.VerificationError` otherwise.
    :rtype: bool
    """
    rv = lib.argon2_verify(
        ffi.new("uint8_t[]", hash),
        ffi.new("uint8_t[]", secret),
        len(secret),
        type.value,
    )
    if rv == lib.ARGON2_OK:
        return True
    else:
        raise VerificationError(error_to_str(rv))

def core(context, type):
    """
    Direct binding to the Argon2 ``core`` function.

    .. warning::
        This is a strictly advanced function working on raw C data structures.
        Both Argon2's and ``argon2_cffi``'s' higher-level bindings do a lot of
        sanity checks and housekeeping work that *you* are now responsible for
        (e.g. clearing buffers).

        Use at your own peril; ``argon2_cffi`` does *not* use this binding
        itself.

    :param context: A CFFI Argon2 context object (i.e. an ``struct
        Argon2_Context``/``argon2_context``).
    :param int type: Which Argon2 variant to use.  You can use the ``value``
        field of :class:`Type`'s fields.

    :rtype: int
    :return: An Argon2 error code.  Can be transformed into a string using
        :func:`error_to_str`.

    .. versionadded:: 16.0.0
    """
    return lib.argon2_core(context, type)

def error_to_str(error):
    """
    Convert an Argon2 error code into a native string.

    :param int error: An Argon2 error code as returned by :func:`core`.

    :rtype: str

    .. versionadded:: 16.0.0
    """
    msg = ffi.string(lib.error_message(error))
    if PY3:
        msg = msg.decode("ascii")
    return msg
<commit_msg>low-level: replace unicode character<commit_after># -*- coding: utf-8 -*-
"""
Low-level functions if you want to build your own higher level abstractions.

.. warning::
    This is a "Hazardous Materials" module.  You should **ONLY** use it if
    you're 100% absolutely sure that you know what you’re doing because this
    module is full of land mines, dragons, and dinosaurs with laser guns.
"""

from __future__ import absolute_import, division, print_function

from enum import Enum

from six import PY3

from ._ffi import ffi, lib
from ._util import _get_encoded_len
from .exceptions import VerificationError, HashingError

__all__ = [
    "Type",
    "ffi",
    "hash_secret",
    "hash_secret_raw",
    "verify_secret",
]

class Type(Enum):
    """
    Enum of Argon2 variants.
    """
    D = lib.Argon2_d
    """
    Argon2\ **d** is faster and uses data-depending memory access, which makes
    it less suitable for hashing secrets and more suitable for cryptocurrencies
    and applications with no threats from side-channel timing attacks.
    """
    I = lib.Argon2_i
    """
    Argon2\ **i** uses data-independent memory access, which is preferred for
    password hashing and password-based key derivation.  Argon2i is slower as
    it makes more passes over the memory to protect from tradeoff attacks.
    """

def hash_secret(secret, salt, time_cost, memory_cost, parallelism, hash_len,
                type):
    """
    Hash *secret* and return an **encoded** hash.

    An encoded hash can be directly passed into :func:`verify_secret` as it
    contains all parameters and the salt.

    :param bytes secret: Secret to hash.
    :param bytes salt: A salt_.  Should be random and different for each
        secret.
    :param Type type: Which Argon2 variant to use.

    For an explanation of the Argon2 parameters see :class:`PasswordHasher`.

    :rtype: bytes

    :raises argon2.exceptions.HashingError: If hashing fails.

    .. versionadded:: 16.0.0

    .. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography)
    .. _kibibytes: https://en.wikipedia.org/wiki/Binary_prefix#kibi
    """
    size = _get_encoded_len(hash_len, len(salt))
    buf = ffi.new("uint8_t[]", size)
    rv = lib.argon2_hash(
        time_cost, memory_cost, parallelism,
        ffi.new("uint8_t[]", secret), len(secret),
        ffi.new("uint8_t[]", salt), len(salt),
        ffi.NULL, hash_len,
        buf, size,
        type.value,
    )
    if rv != lib.ARGON2_OK:
        raise HashingError(error_to_str(rv))

    return ffi.string(buf)

def hash_secret_raw(secret, salt, time_cost, memory_cost, parallelism,
                    hash_len, type):
    """
    Hash *password* and return a **raw** hash.

    This function takes the same parameters as :func:`hash_secret`.
    """
    buf = ffi.new("uint8_t[]", hash_len)

    rv = lib.argon2_hash(
        time_cost, memory_cost, parallelism,
        ffi.new("uint8_t[]", secret), len(secret),
        ffi.new("uint8_t[]", salt), len(salt),
        buf, hash_len,
        ffi.NULL, 0,
        type.value,
    )
    if rv != lib.ARGON2_OK:
        raise HashingError(error_to_str(rv))

    return bytes(ffi.buffer(buf, hash_len))

def verify_secret(hash, secret, type):
    """
    Verify whether *secret* is correct for *hash* of *type*.

    :param bytes hash: An encoded Argon2 hash as returned by
        :func:`hash_secret`.
    :param bytes secret: The secret to verify whether it matches the one
        in *hash*.
    :param Type type: Type for *hash*.

    :raises argon2.exceptions.VerificationError: If verification fails.

    :return: ``True`` on success, raise
            :exc:`~argon2.exceptions.VerificationError` otherwise.
    :rtype: bool
    """
    rv = lib.argon2_verify(
        ffi.new("uint8_t[]", hash),
        ffi.new("uint8_t[]", secret),
        len(secret),
        type.value,
    )
    if rv == lib.ARGON2_OK:
        return True
    else:
        raise VerificationError(error_to_str(rv))

def core(context, type):
    """
    Direct binding to the Argon2 ``core`` function.

    .. warning::
        This is a strictly advanced function working on raw C data structures.
        Both Argon2's and ``argon2_cffi``'s' higher-level bindings do a lot of
        sanity checks and housekeeping work that *you* are now responsible for
        (e.g. clearing buffers).

        Use at your own peril; ``argon2_cffi`` does *not* use this binding
        itself.

    :param context: A CFFI Argon2 context object (i.e. an ``struct
        Argon2_Context``/``argon2_context``).
    :param int type: Which Argon2 variant to use.  You can use the ``value``
        field of :class:`Type`'s fields.

    :rtype: int
    :return: An Argon2 error code.  Can be transformed into a string using
        :func:`error_to_str`.

    .. versionadded:: 16.0.0
    """
    return lib.argon2_core(context, type)

def error_to_str(error):
    """
    Convert an Argon2 error code into a native string.

    :param int error: An Argon2 error code as returned by :func:`core`.

    :rtype: str

    .. versionadded:: 16.0.0
    """
    msg = ffi.string(lib.error_message(error))
    if PY3:
        msg = msg.decode("ascii")
    return msg
<|endoftext|>