<commit_before>#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# common.py

""" Common code used by purify_test.py and quantify_test.py in order to automate
running of Rational Purify and Quantify in a consistent manner.
"""

# Purify and Quantify have a front-end (e.g. quantifyw.exe) which talks to a
# back-end engine (e.g. quantifye.exe).  The back-end seems to handle
# instrumentation, while the front-end controls program execution and
# measurement.  The front-end will dynamically launch the back-end if
# instrumentation is needed (sometimes in the middle of a run if a dll is
# loaded dynamically).
# In an ideal world, this script would simply execute the front-end and check
# the output.  However, purify is not the most reliable or well-documented app
# on the planet, and my attempts to get it to run this way led to the back-end
# engine hanging during instrumentation.  The workaround to this was to run two
# passes, first running the engine to do instrumentation rather than letting
# the front-end do it for you, then running the front-end to actually do the
# run.  Each time through we're deleting all of the instrumented files in the
# cache to ensure that we're testing that instrumentation works from scratch.
# (although this can be changed with an option)

import datetime
import logging
import optparse
import os
import subprocess
import sys
import tempfile
import time

import google.logging_utils

# hard-coded location of Rational files and directories
RATIONAL_PATH = os.path.join("C:\\", "Program Files", "Rational")
COMMON_PATH = os.path.join(RATIONAL_PATH, "common")
PPLUS_PATH = os.path.join(RATIONAL_PATH, "PurifyPlus")
PURIFY_PATH = os.path.join(COMMON_PATH, "purify.exe")
PURIFYW_PATH = os.path.join(PPLUS_PATH, "purifyW.exe")
PURIFYE_PATH = os.path.join(PPLUS_PATH, "purifye.exe")
QUANTIFYE_PATH = os.path.join(PPLUS_PATH, "quantifye.exe")
QUANTIFYW_PATH = os.path.join(PPLUS_PATH, "quantifyw.exe")

class TimeoutError(Exception): pass

def _print_line(line, flush=True):
  # Printing to a text file (including stdout) on Windows always winds up
  # using \r\n automatically.  On buildbot, this winds up being read by a master
  # running on Linux, so this is a pain.  Unfortunately, it doesn't matter what
  # we do here, so just leave this comment for future reference.
  print line,
  if flush:
    sys.stdout.flush()

def RunSubprocess(proc, timeout=0, detach=False):
  """ Runs a subprocess, polling every .2 seconds until it finishes or until
  timeout is reached.  Then kills the process with taskkill.  A timeout <= 0
  means no timeout.

  Args:
    proc: list of process components (exe + args)
    timeout: how long to wait before killing, <= 0 means wait forever
    detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
        on Windows.  This is used by Purify subprocesses on buildbot which
        seem to get confused by the parent console that buildbot sets up.
  """
  logging.info("running %s" % (" ".join(proc)))
  if detach:
    # see MSDN docs for "Process Creation Flags"
    DETACHED_PROCESS = 0x8
    p = subprocess.Popen(proc, creationflags=DETACHED_PROCESS)
  else:
    # For non-detached processes, manually read and print out stdout and stderr.
    # By default, the subprocess is supposed to inherit these from its parent,
    # however when run under buildbot, it seems unable to read data from a
    # grandchild process, so we have to read the child and print the data as if
    # it came from us for buildbot to read it.  We're not sure why this is
    # necessary.
    # TODO(erikkay): should we buffer stderr and stdout separately?
    p = subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  if timeout <= 0:
    while p.poll() is None:
      if not detach:
        line = p.stdout.readline()
        while line:
          _print_line(line)
          line = p.stdout.readline()
      time.sleep(0.2)
  else:
    wait_until = time.time() + timeout
    while p.poll() is None and time.time() < wait_until:
      if not detach:
        line = p.stdout.readline()
        while line:
          _print_line(line)
          line = p.stdout.readline()
      time.sleep(0.2)
  if not detach:
    for line in p.stdout.readlines():
      _print_line(line, False)
    p.stdout.flush()
  result = p.poll()
  if result is None:
    subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
    logging.error("KILLED %d" % p.pid)
    # give the process a chance to actually die before continuing
    # so that cleanup can happen safely
    time.sleep(1.0)
    logging.error("TIMEOUT waiting for %s" % proc[0])
    raise TimeoutError(proc[0])
  if result:
    logging.error("%s exited with non-zero result code %d" % (proc[0], result))
  return result

def FixPath(path):
  """We pass computed paths to Rational as arguments, so these paths must be
  valid windows paths.  When running in cygwin's python, computed paths
  wind up looking like /cygdrive/c/..., so we need to call out to cygpath
  to fix them up.
  """
  if sys.platform != "cygwin":
    return path
  p = subprocess.Popen(["cygpath", "-a", "-m", path], stdout=subprocess.PIPE)
  return p.communicate()[0].rstrip()

class Rational(object):
  ''' Common superclass for Purify and Quantify automation objects.  Handles
  common argument parsing as well as the general program flow of Instrument,
  Execute, Analyze.
  '''

  def __init__(self):
    google.logging_utils.config_root()
    self._out_file = None

  def Run(self):
    '''Call this to run through the whole process:
    Setup, Instrument, Execute, Analyze'''
    start = datetime.datetime.now()
    retcode = -1
    if self.Setup():
      retcode = self._Run()
      self.Cleanup()
    else:
      logging.error("Setup failed")
    end = datetime.datetime.now()
    seconds = (end - start).seconds
    hours = seconds / 3600
    seconds = seconds % 3600
    minutes = seconds / 60
    seconds = seconds % 60
    logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds))
    return retcode

  def _Run(self):
    retcode = -1
    if not self.Instrument():
      logging.error("Instrumentation failed.")
      return retcode
    if self._instrument_only:
      logging.info("Instrumentation completed successfully.")
      return 0
    if not self.Execute():
      logging.error("Execute failed.")
      return
    retcode = self.Analyze()
    if retcode:
      logging.error("Analyze failed.")
      return retcode
    logging.info("Instrumentation and execution completed successfully.")
    return 0

  def CreateOptionParser(self):
    '''Creates OptionParser with shared arguments.  Overridden by subclassers
    to add custom arguments.'''
    parser = optparse.OptionParser("usage: %prog [options] <program to test>")
    # since the trailing program likely has command-line args of itself
    # we need to stop parsing when we reach the first positional arg
    parser.disable_interspersed_args()
    parser.add_option("-o", "--out_file", dest="out_file", metavar="OUTFILE",
                      default="",
                      help="output data is written to OUTFILE")
    parser.add_option("-s", "--save_cache",
                      dest="save_cache", action="store_true", default=False,
                      help="don't delete instrumentation cache")
    parser.add_option("-c", "--cache_dir", dest="cache_dir", metavar="CACHEDIR",
                      default="",
                      help="location of instrumentation cache is CACHEDIR")
    parser.add_option("-m", "--manual",
                      dest="manual_run", action="store_true", default=False,
                      help="target app is being run manually, don't timeout")
    parser.add_option("-t", "--timeout",
                      dest="timeout", metavar="TIMEOUT", default=10000,
                      help="timeout in seconds for the run (default 10000)")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="verbose output - enable debug log messages")
    parser.add_option("", "--instrument_only", action="store_true",
                      default=False,
                      help="Only instrument the target without running")
    self._parser = parser

  def Setup(self):
    if self.ParseArgv():
      logging.info("instrumentation cache in %s" % self._cache_dir)
      logging.info("output saving to %s" % self._out_file)
      # Ensure that Rational's common dir and cache dir are in the front of the
      # path.  The common dir is required for purify to run in any case, and
      # the cache_dir is required when using the /Replace=yes option.
      os.environ["PATH"] = (COMMON_PATH + ";" + self._cache_dir + ";" +
          os.environ["PATH"])
      # clear the cache to make sure we're starting clean
      self.__ClearInstrumentationCache()
      return True
    return False

  def Instrument(self, proc):
    '''Instrument the app to be tested.  Full instrumentation command-line
    provided by subclassers via proc.'''
    logging.info("starting instrumentation...")
    if RunSubprocess(proc, self._timeout, detach=True) == 0:
      if "/Replace=yes" in proc:
        if os.path.exists(self._exe + ".Original"):
          return True
      elif self._instrument_only:
        # TODO(paulg): Catch instrumentation errors and clean up properly.
        return True
      elif os.path.isdir(self._cache_dir):
        for cfile in os.listdir(self._cache_dir):
          # TODO(erikkay): look for the actual munged purify filename
          ext = os.path.splitext(cfile)[1]
          if ext == ".exe":
            return True
      logging.error("no instrumentation data generated")
    return False

  def Execute(self, proc):
    ''' Execute the app to be tested after successful instrumentation.
    Full execution command-line provided by subclassers via proc.'''
    logging.info("starting execution...")
    # note that self._args begins with the exe to be run
    proc += self._args
    if RunSubprocess(proc, self._timeout) == 0:
      return True
    return False

  def Analyze(self):
    '''Analyze step after a successful Execution.  Should be overridden
    by the subclasser if instrumentation is desired.
    Returns 0 for success, -88 for warning (see ReturnCodeCommand) and anything
    else for error
    '''
    return -1

  def ParseArgv(self):
    '''Parses arguments according to CreateOptionParser
    Subclassers must override if they have extra arguments.'''
    self.CreateOptionParser()
    self._options, self._args = self._parser.parse_args()
    if self._options.verbose:
      google.logging_utils.config_root(logging.DEBUG)
    self._save_cache = self._options.save_cache
    self._manual_run = self._options.manual_run
    if self._manual_run:
      logging.info("manual run - timeout disabled")
      self._timeout = 0
    else:
      self._timeout = int(self._options.timeout)
      logging.info("timeout set to %ds" % (self._timeout))
    if self._save_cache:
      logging.info("saving instrumentation cache")
    if not self._options.cache_dir:
      try:
        temp_dir = os.environ["TEMP"]
      except KeyError:
        temp_dir = tempfile.mkdtemp()
      self._cache_dir = os.path.join(FixPath(temp_dir),
                                     "instrumentation_cache")
    else:
      self._cache_dir = FixPath(os.path.abspath(self._options.cache_dir))
    if self._options.out_file:
      self._out_file = FixPath(os.path.abspath(self._options.out_file))
    if len(self._args) == 0:
      self._parser.error("missing program to %s" % (self.__class__.__name__,))
      return False
    self._exe = self._args[0]
    self._exe_dir = FixPath(os.path.abspath(os.path.dirname(self._exe)))
    self._instrument_only = self._options.instrument_only
    return True

  def Cleanup(self):
    # delete the cache to avoid filling up the hard drive when we're using
    # temporary directory names
    self.__ClearInstrumentationCache()

  def __ClearInstrumentationCache(self):
    if not self._save_cache:
      logging.info("clearing instrumentation cache %s" % self._cache_dir)
      if os.path.isdir(self._cache_dir):
        for cfile in os.listdir(self._cache_dir):
          file = os.path.join(self._cache_dir, cfile);
          if os.path.isfile(file):
            try:
              os.remove(file)
            except:
              logging.warning("unable to delete file %s: %s" % (file,
                              sys.exc_info()[0]))

<commit_msg>Fix the output line endinf<commit_after>#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# common.py

""" Common code used by purify_test.py and quantify_test.py in order to automate
running of Rational Purify and Quantify in a consistent manner.
"""

# Purify and Quantify have a front-end (e.g. quantifyw.exe) which talks to a
# back-end engine (e.g. quantifye.exe).  The back-end seems to handle
# instrumentation, while the front-end controls program execution and
# measurement.  The front-end will dynamically launch the back-end if
# instrumentation is needed (sometimes in the middle of a run if a dll is
# loaded dynamically).
# In an ideal world, this script would simply execute the front-end and check
# the output.  However, purify is not the most reliable or well-documented app
# on the planet, and my attempts to get it to run this way led to the back-end
# engine hanging during instrumentation.  The workaround to this was to run two
# passes, first running the engine to do instrumentation rather than letting
# the front-end do it for you, then running the front-end to actually do the
# run.  Each time through we're deleting all of the instrumented files in the
# cache to ensure that we're testing that instrumentation works from scratch.
# (although this can be changed with an option)

import datetime
import logging
import optparse
import os
import subprocess
import sys
import tempfile
import time

import google.logging_utils

# hard-coded location of Rational files and directories
RATIONAL_PATH = os.path.join("C:\\", "Program Files", "Rational")
COMMON_PATH = os.path.join(RATIONAL_PATH, "common")
PPLUS_PATH = os.path.join(RATIONAL_PATH, "PurifyPlus")
PURIFY_PATH = os.path.join(COMMON_PATH, "purify.exe")
PURIFYW_PATH = os.path.join(PPLUS_PATH, "purifyW.exe")
PURIFYE_PATH = os.path.join(PPLUS_PATH, "purifye.exe")
QUANTIFYE_PATH = os.path.join(PPLUS_PATH, "quantifye.exe")
QUANTIFYW_PATH = os.path.join(PPLUS_PATH, "quantifyw.exe")

class TimeoutError(Exception): pass

def _print_line(line, flush=True):
  # Printing to a text file (including stdout) on Windows always winds up
  # using \r\n automatically.  On buildbot, this winds up being read by a master
  # running on Linux, so this is a pain.  Unfortunately, it doesn't matter what
  # we do here, so just leave this comment for future reference.
  print line.rstrip() + '\n',
  if flush:
    sys.stdout.flush()

def RunSubprocess(proc, timeout=0, detach=False):
  """ Runs a subprocess, polling every .2 seconds until it finishes or until
  timeout is reached.  Then kills the process with taskkill.  A timeout <= 0
  means no timeout.

  Args:
    proc: list of process components (exe + args)
    timeout: how long to wait before killing, <= 0 means wait forever
    detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
        on Windows.  This is used by Purify subprocesses on buildbot which
        seem to get confused by the parent console that buildbot sets up.
  """
  logging.info("running %s" % (" ".join(proc)))
  if detach:
    # see MSDN docs for "Process Creation Flags"
    DETACHED_PROCESS = 0x8
    p = subprocess.Popen(proc, creationflags=DETACHED_PROCESS)
  else:
    # For non-detached processes, manually read and print out stdout and stderr.
    # By default, the subprocess is supposed to inherit these from its parent,
    # however when run under buildbot, it seems unable to read data from a
    # grandchild process, so we have to read the child and print the data as if
    # it came from us for buildbot to read it.  We're not sure why this is
    # necessary.
    # TODO(erikkay): should we buffer stderr and stdout separately?
    p = subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  if timeout <= 0:
    while p.poll() is None:
      if not detach:
        line = p.stdout.readline()
        while line:
          _print_line(line)
          line = p.stdout.readline()
      time.sleep(0.2)
  else:
    wait_until = time.time() + timeout
    while p.poll() is None and time.time() < wait_until:
      if not detach:
        line = p.stdout.readline()
        while line:
          _print_line(line)
          line = p.stdout.readline()
      time.sleep(0.2)
  if not detach:
    for line in p.stdout.readlines():
      _print_line(line, False)
    p.stdout.flush()
  result = p.poll()
  if result is None:
    subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
    logging.error("KILLED %d" % p.pid)
    # give the process a chance to actually die before continuing
    # so that cleanup can happen safely
    time.sleep(1.0)
    logging.error("TIMEOUT waiting for %s" % proc[0])
    raise TimeoutError(proc[0])
  if result:
    logging.error("%s exited with non-zero result code %d" % (proc[0], result))
  return result

def FixPath(path):
  """We pass computed paths to Rational as arguments, so these paths must be
  valid windows paths.  When running in cygwin's python, computed paths
  wind up looking like /cygdrive/c/..., so we need to call out to cygpath
  to fix them up.
  """
  if sys.platform != "cygwin":
    return path
  p = subprocess.Popen(["cygpath", "-a", "-m", path], stdout=subprocess.PIPE)
  return p.communicate()[0].rstrip()

class Rational(object):
  ''' Common superclass for Purify and Quantify automation objects.  Handles
  common argument parsing as well as the general program flow of Instrument,
  Execute, Analyze.
  '''

  def __init__(self):
    google.logging_utils.config_root()
    self._out_file = None

  def Run(self):
    '''Call this to run through the whole process:
    Setup, Instrument, Execute, Analyze'''
    start = datetime.datetime.now()
    retcode = -1
    if self.Setup():
      retcode = self._Run()
      self.Cleanup()
    else:
      logging.error("Setup failed")
    end = datetime.datetime.now()
    seconds = (end - start).seconds
    hours = seconds / 3600
    seconds = seconds % 3600
    minutes = seconds / 60
    seconds = seconds % 60
    logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds))
    return retcode

  def _Run(self):
    retcode = -1
    if not self.Instrument():
      logging.error("Instrumentation failed.")
      return retcode
    if self._instrument_only:
      logging.info("Instrumentation completed successfully.")
      return 0
    if not self.Execute():
      logging.error("Execute failed.")
      return
    retcode = self.Analyze()
    if retcode:
      logging.error("Analyze failed.")
      return retcode
    logging.info("Instrumentation and execution completed successfully.")
    return 0

  def CreateOptionParser(self):
    '''Creates OptionParser with shared arguments.  Overridden by subclassers
    to add custom arguments.'''
    parser = optparse.OptionParser("usage: %prog [options] <program to test>")
    # since the trailing program likely has command-line args of itself
    # we need to stop parsing when we reach the first positional arg
    parser.disable_interspersed_args()
    parser.add_option("-o", "--out_file", dest="out_file", metavar="OUTFILE",
                      default="",
                      help="output data is written to OUTFILE")
    parser.add_option("-s", "--save_cache",
                      dest="save_cache", action="store_true", default=False,
                      help="don't delete instrumentation cache")
    parser.add_option("-c", "--cache_dir", dest="cache_dir", metavar="CACHEDIR",
                      default="",
                      help="location of instrumentation cache is CACHEDIR")
    parser.add_option("-m", "--manual",
                      dest="manual_run", action="store_true", default=False,
                      help="target app is being run manually, don't timeout")
    parser.add_option("-t", "--timeout",
                      dest="timeout", metavar="TIMEOUT", default=10000,
                      help="timeout in seconds for the run (default 10000)")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="verbose output - enable debug log messages")
    parser.add_option("", "--instrument_only", action="store_true",
                      default=False,
                      help="Only instrument the target without running")
    self._parser = parser

  def Setup(self):
    if self.ParseArgv():
      logging.info("instrumentation cache in %s" % self._cache_dir)
      logging.info("output saving to %s" % self._out_file)
      # Ensure that Rational's common dir and cache dir are in the front of the
      # path.  The common dir is required for purify to run in any case, and
      # the cache_dir is required when using the /Replace=yes option.
      os.environ["PATH"] = (COMMON_PATH + ";" + self._cache_dir + ";" +
          os.environ["PATH"])
      # clear the cache to make sure we're starting clean
      self.__ClearInstrumentationCache()
      return True
    return False

  def Instrument(self, proc):
    '''Instrument the app to be tested.  Full instrumentation command-line
    provided by subclassers via proc.'''
    logging.info("starting instrumentation...")
    if RunSubprocess(proc, self._timeout, detach=True) == 0:
      if "/Replace=yes" in proc:
        if os.path.exists(self._exe + ".Original"):
          return True
      elif self._instrument_only:
        # TODO(paulg): Catch instrumentation errors and clean up properly.
        return True
      elif os.path.isdir(self._cache_dir):
        for cfile in os.listdir(self._cache_dir):
          # TODO(erikkay): look for the actual munged purify filename
          ext = os.path.splitext(cfile)[1]
          if ext == ".exe":
            return True
      logging.error("no instrumentation data generated")
    return False

  def Execute(self, proc):
    ''' Execute the app to be tested after successful instrumentation.
    Full execution command-line provided by subclassers via proc.'''
    logging.info("starting execution...")
    # note that self._args begins with the exe to be run
    proc += self._args
    if RunSubprocess(proc, self._timeout) == 0:
      return True
    return False

  def Analyze(self):
    '''Analyze step after a successful Execution.  Should be overridden
    by the subclasser if instrumentation is desired.
    Returns 0 for success, -88 for warning (see ReturnCodeCommand) and anything
    else for error
    '''
    return -1

  def ParseArgv(self):
    '''Parses arguments according to CreateOptionParser
    Subclassers must override if they have extra arguments.'''
    self.CreateOptionParser()
    self._options, self._args = self._parser.parse_args()
    if self._options.verbose:
      google.logging_utils.config_root(logging.DEBUG)
    self._save_cache = self._options.save_cache
    self._manual_run = self._options.manual_run
    if self._manual_run:
      logging.info("manual run - timeout disabled")
      self._timeout = 0
    else:
      self._timeout = int(self._options.timeout)
      logging.info("timeout set to %ds" % (self._timeout))
    if self._save_cache:
      logging.info("saving instrumentation cache")
    if not self._options.cache_dir:
      try:
        temp_dir = os.environ["TEMP"]
      except KeyError:
        temp_dir = tempfile.mkdtemp()
      self._cache_dir = os.path.join(FixPath(temp_dir),
                                     "instrumentation_cache")
    else:
      self._cache_dir = FixPath(os.path.abspath(self._options.cache_dir))
    if self._options.out_file:
      self._out_file = FixPath(os.path.abspath(self._options.out_file))
    if len(self._args) == 0:
      self._parser.error("missing program to %s" % (self.__class__.__name__,))
      return False
    self._exe = self._args[0]
    self._exe_dir = FixPath(os.path.abspath(os.path.dirname(self._exe)))
    self._instrument_only = self._options.instrument_only
    return True

  def Cleanup(self):
    # delete the cache to avoid filling up the hard drive when we're using
    # temporary directory names
    self.__ClearInstrumentationCache()

  def __ClearInstrumentationCache(self):
    if not self._save_cache:
      logging.info("clearing instrumentation cache %s" % self._cache_dir)
      if os.path.isdir(self._cache_dir):
        for cfile in os.listdir(self._cache_dir):
          file = os.path.join(self._cache_dir, cfile);
          if os.path.isfile(file):
            try:
              os.remove(file)
            except:
              logging.warning("unable to delete file %s: %s" % (file,
                              sys.exc_info()[0]))

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# pylint: disable=redefined-outer-name

"""" Gaussian Process Regression model

Implementation is based on Algorithm 2.1 (pg. 19) of
"Gaussian Processes for Machine Learning" by Rasmussen and Williams

Simon Batzner, Jon V
"""

import math

import numpy as np
from scipy.linalg import solve_triangular
from scipy.optimize import minimize
from typing import List

from env import ChemicalEnvironment, two_body, two_body_py, three_body,\
    three_body_py
from struc import Structure

class GaussianProcess:
    """ Gaussian Process Regression Model """

    def __init__(self, kernel: str):
        """Initialize GP parameters and training data.

        :param kernel: covariance/ kernel function used
        :type kernel: str
        """

        # gp kernel and hyperparameters
        self.kernel_type = kernel
        self.length_scale = 1
        self.sigma_n = 1
        self.sigma_f = 0.01

        # quantities used in GPR algorithm
        self.k_mat = None
        self.l_mat = None
        self.alpha = None

        # training set
        self.training_data = []
        self.training_labels = []
        self.training_labels_np = np.empty(0,)

    def kernel(self, env1: ChemicalEnvironment, env2: ChemicalEnvironment,
               d1: int, d2: int, sig: float, ls: float):
        """Evaluate specified kernel."""

        if self.kernel_type == 'two_body':
            return two_body(env1, env2, d1, d2, sig, ls)

        elif self.kernel_type == 'two_body_py':
            return two_body_py(env1, env2, d1, d2, sig, ls)

        elif self.kernel_type == 'three_body':
            return three_body(env1, env2, d1, d2, sig, ls)

        elif self.kernel_type == 'three_body_py':
            return three_body_py(env1, env2, d1, d2, sig, ls)

        else:
            raise ValueError('{} is not a valid kernel'.format(self.kernel))

    def update_db(self, struc: Structure, forces: list):
        """Given structure and forces, add to training set.

        :param struc: [description]
        :type struc: [type]
        :param forces: [description]
        :type forces: [type]
        """

        noa = len(struc.positions)

        for atom in range(noa):
            env_curr = ChemicalEnvironment(struc, atom)
            forces_curr = forces[atom]

            self.training_data.append(env_curr)
            self.training_labels.append(forces_curr)

        # create numpy array of training labels
        self.training_labels_np = self.force_list_to_np(self.training_labels)

    @staticmethod
    def force_list_to_np(forces: list) -> np.ndarray:
        forces_np = []
        for m in range(len(forces)):
            for n in range(3):
                forces_np.append(forces[m][n])
        forces_np = np.array(forces_np)
        return forces_np

    def train(self):
        """ Train Gaussian Process model on training data """

        # optimize hyperparameters
        self.opt_hyper()

        # following: Algorithm 2.1 (pg. 19) of
        # "Gaussian Processes for Machine Learning" by Rasmussen and Williams
        self.set_kernel(sigma_f=self.sigma_f,
                        length_scale=self.length_scale,
                        sigma_n=self.sigma_n)

        # get alpha and likelihood
        self.set_alpha()

    def opt_hyper(self):
        """
        Optimize hyperparameters of GP by minimizing minus log likelihood
        """
        # initial guess
        x_0 = np.array([self.sigma_f, self.length_scale, self.sigma_n])

        # nelder-mead optimization
        args = (self,)
        res = minimize(minus_like_hyp, x_0, args,
                       method='nelder-mead',
                       options={'xtol': 1e-8, 'disp': False})

        self.sigma_f = res.x[0]
        self.length_scale = res.x[1]
        self.sigma_n = res.x[2]

    def predict(self, x_t: ChemicalEnvironment, d: int) -> [float, float]:
        """Make GP prediction with SE kernel

        :param x_t: data point to predict on
        :type x_t:
        :param d: kernel parameter
        :type d:
        """

        # get kernel vector
        k_v = self.get_kernel_vector(x_t, d)

        # get predictive mean
        pred_mean = np.matmul(k_v, self.alpha)

        # get predictive variance
        v_vec = solve_triangular(self.l_mat, k_v, lower=True)
        self_kern = self.kernel(x_t, x_t, d, d,
                                self.sigma_f, self.length_scale)
        pred_var = self_kern - np.matmul(v_vec, v_vec)

        return pred_mean, pred_var

    def like_hyp(self, hyp: List[float]) -> float:
        """ Get likelihood as a function of hyperparameters

        :param hyp: hyperparameters to optimize
        :type hyp: list<float>
        :return: likelihood
        :rtype: float
        """

        # unpack hyperparameters
        sigma_f = hyp[0]
        length_scale = hyp[1]
        sigma_n = hyp[2]

        # calculate likelihood
        self.set_kernel(sigma_f=sigma_f,
                        length_scale=length_scale,
                        sigma_n=sigma_n)

        self.set_alpha()
        like = self.get_likelihood()

        return like

    def get_likelihood(self) -> float:
        """
        Get log marginal likelihood

        :return: likelihood
        :rtype:
        """
        like = np.asscalar(-(1 / 2) *
                           np.matmul(self.training_labels_np.transpose(),
                                     self.alpha) -
                           np.sum(np.log(np.diagonal(self.l_mat))) -
                           np.log(2 * np.pi) * self.k_mat.shape[1] / 2)

        return like

    def set_kernel(self, sigma_f: float, length_scale: float, sigma_n: float):
        """ Compute 3Nx3N noiseless kernel matrix

        :param sigma_f: signal variance
        :type sigma_f: float
        :param length_scale: length scale of the GP
        :type length_scale: float
        :param sigma_n: noise variance
        :type sigma_n: float

        """

        # initialize matrix
        size = len(self.training_data)*3
        k_mat = np.zeros([size, size])

        ds = [1, 2, 3]

        # calculate elements
        for m_index in range(size):
            x_1 = self.training_data[int(math.floor(m_index / 3))]
            d_1 = ds[m_index % 3]

            for n_index in range(m_index, size):
                x_2 = self.training_data[int(math.floor(n_index / 3))]
                d_2 = ds[n_index % 3]

                # calculate kernel
                cov = self.kernel(x_1, x_2, d_1, d_2, sigma_f, length_scale)
                k_mat[m_index, n_index] = cov
                k_mat[n_index, m_index] = cov

        # perform cholesky decomposition and store
        self.l_mat = np.linalg.cholesky(k_mat + sigma_n ** 2 * np.eye(size))
        self.k_mat = k_mat

    def get_kernel_vector(self, x: ChemicalEnvironment, d_1: int)-> np.ndarray:
        """ Compute kernel vector

        :param x: data point to compare against kernel matrix
        :type x:
        :param d_1:
        :type d_1:
        :return: kernel vector
        :rtype:
        """
        ds = [1, 2, 3]
        size = len(self.training_data)*3
        k_v = np.zeros(size,)

        for m_index in range(size):
            x_2 = self.training_data[int(math.floor(m_index/3))]
            d_2 = ds[m_index % 3]
            k_v[m_index] = self.kernel(x, x_2, d_1, d_2,
                                       self.sigma_f, self.length_scale)

        return k_v

    def set_alpha(self):
        """ Set weight vector alpha """
        ts1 = solve_triangular(self.l_mat, self.training_labels_np, lower=True)
        self.alpha = solve_triangular(self.l_mat.transpose(), ts1)

def minus_like_hyp(hyp: List[float],
                   gp: GaussianProcess,
                   verbose: bool = True) -> float:
    """Get minus likelihood as a function of hyperparameters"""
    like = gp.like_hyp(hyp)
    minus_like = -like

    if verbose:
        print(hyp)
        print(like)

    return minus_like

<commit_msg>updated docs<commit_after>#!/usr/bin/env python3
# pylint: disable=redefined-outer-name

"""" Gaussian Process Regression model

Implementation is based on Algorithm 2.1 (pg. 19) of
"Gaussian Processes for Machine Learning" by Rasmussen and Williams

Simon Batzner, Jon Vandermause
"""

import math

import numpy as np
from scipy.linalg import solve_triangular
from scipy.optimize import minimize
from typing import List

from env import ChemicalEnvironment, two_body, two_body_py, three_body,\
    three_body_py
from struc import Structure

class GaussianProcess:
    """ Gaussian Process Regression Model """

    def __init__(self, kernel: str):
        """Initialize GP parameters and training data.

        :param kernel: covariance / kernel function to be used
        :type kernel: str
        """

        # gp kernel and hyperparameters
        self.kernel_type = kernel
        self.length_scale = 1
        self.sigma_n = 1
        self.sigma_f = 0.01

        # quantities used in GPR algorithm
        self.k_mat = None
        self.l_mat = None
        self.alpha = None

        # training set
        self.training_data = []
        self.training_labels = []
        self.training_labels_np = np.empty(0,)

    def kernel(self, env1: ChemicalEnvironment, env2: ChemicalEnvironment,
               d1: int, d2: int, sig: float, length_scale: float):
        """Evaluate specified kernel.

        :param env1: first chemical environment to compare
        :type evn1: ChemicalEnvironment
        :param env2: second chemical environment to compare
        :type env2: ChemicalEnvironment
        :param d1:
        :type d1: int
        :param d2:
        :type d2: int
        :param sig: signal variance
        :type sig: float
        :param length_scale: length scale of kernel
        :type: length_scale: float
        """

        if self.kernel_type == 'two_body':
            return two_body(env1, env2, d1, d2, sig, length_scale)

        elif self.kernel_type == 'two_body_py':
            return two_body_py(env1, env2, d1, d2, sig, length_scale)

        elif self.kernel_type == 'three_body':
            return three_body(env1, env2, d1, d2, sig, length_scale)

        elif self.kernel_type == 'three_body_py':
            return three_body_py(env1, env2, d1, d2, sig, length_scale)

        else:
            raise ValueError('{} is not a valid kernel'.format(self.kernel))

    def update_db(self, struc: Structure, forces: list):
        """Given structure and forces, add to training set.

        :param struc: structure to add to db
        :type struc: Structure
        :param forces: list of corresponding forces to add to db
        :type forces: list<float>
        """

        noa = len(struc.positions)

        for atom in range(noa):
            env_curr = ChemicalEnvironment(struc, atom)
            forces_curr = forces[atom]

            self.training_data.append(env_curr)
            self.training_labels.append(forces_curr)

        # create numpy array of training labels
        self.training_labels_np = self.force_list_to_np(self.training_labels)

    @staticmethod
    def force_list_to_np(forces: list) -> np.ndarray:
        """ Convert list of forces to numpy array of forces.

        :param forces: list of forces to convert
        :type forces: list<float>
        :return: numpy array forces
        :rtype: np.ndarray
        """
        forces_np = []

        for m in range(len(forces)):
            for n in range(3):
                forces_np.append(forces[m][n])

        forces_np = np.array(forces_np)

        return forces_np

    def train(self):
        """ Train Gaussian Process model on training data. """

        # optimize hyperparameters
        self.opt_hyper()

        # following: Algorithm 2.1 (pg. 19) of
        # "Gaussian Processes for Machine Learning" by Rasmussen and Williams
        self.set_kernel(sigma_f=self.sigma_f,
                        length_scale=self.length_scale,
                        sigma_n=self.sigma_n)

        # get alpha and likelihood
        self.set_alpha()

    def opt_hyper(self):
        """
        Optimize hyperparameters of GP by minimizing minus log likelihood. """
        # initial guess
        x_0 = np.array([self.sigma_f, self.length_scale, self.sigma_n])

        # nelder-mead optimization
        args = (self,)
        res = minimize(minus_like_hyp, x_0, args,
                       method='nelder-mead',
                       options={'xtol': 1e-8, 'disp': False})

        self.sigma_f = res.x[0]
        self.length_scale = res.x[1]
        self.sigma_n = res.x[2]

    def predict(self, x_t: ChemicalEnvironment, d: int) -> [float, float]:
        """ Make GP prediction with SE kernel.

        :param x_t: data point to predict on
        :type x_t: ChemicalEnvironment
        :param d:
        :type d: int
        :return: predictive mean and predictive variance
        :rtype: [float, float]
        """

        # get kernel vector
        k_v = self.get_kernel_vector(x_t, d)

        # get predictive mean
        pred_mean = np.matmul(k_v, self.alpha)

        # get predictive variance
        v_vec = solve_triangular(self.l_mat, k_v, lower=True)
        self_kern = self.kernel(x_t, x_t, d, d,
                                self.sigma_f, self.length_scale)
        pred_var = self_kern - np.matmul(v_vec, v_vec)

        return pred_mean, pred_var

    def like_hyp(self, hyp: List[float]) -> float:
        """ Get likelihood as a function of hyperparameters.

        :param hyp: hyperparameters to optimize
        :type hyp: list<float>
        :return: likelihood
        :rtype: float
        """

        # unpack hyperparameters
        sigma_f = hyp[0]
        length_scale = hyp[1]
        sigma_n = hyp[2]

        # calculate likelihood
        self.set_kernel(sigma_f=sigma_f,
                        length_scale=length_scale,
                        sigma_n=sigma_n)

        self.set_alpha()
        like = self.get_likelihood()

        return like

    def get_likelihood(self) -> float:
        """ Get log marginal likelihood.

        :return: likelihood
        :rtype: float
        """
        like = np.asscalar(-(1 / 2) *
                           np.matmul(self.training_labels_np.transpose(),
                                     self.alpha) -
                           np.sum(np.log(np.diagonal(self.l_mat))) -
                           np.log(2 * np.pi) * self.k_mat.shape[1] / 2)

        return like

    def set_kernel(self, sigma_f: float, length_scale: float, sigma_n: float):
        """ Compute 3Nx3N noiseless kernel matrix.

        :param sigma_f: signal variance
        :type sigma_f: float
        :param length_scale: length scale of the GP
        :type length_scale: float
        :param sigma_n: noise variance
        :type sigma_n: float

        """

        # initialize matrix
        size = len(self.training_data)*3
        k_mat = np.zeros([size, size])

        ds = [1, 2, 3]

        # calculate elements
        for m_index in range(size):
            x_1 = self.training_data[int(math.floor(m_index / 3))]
            d_1 = ds[m_index % 3]

            for n_index in range(m_index, size):
                x_2 = self.training_data[int(math.floor(n_index / 3))]
                d_2 = ds[n_index % 3]

                # calculate kernel
                cov = self.kernel(x_1, x_2, d_1, d_2, sigma_f, length_scale)
                k_mat[m_index, n_index] = cov
                k_mat[n_index, m_index] = cov

        # perform cholesky decomposition and store
        self.l_mat = np.linalg.cholesky(k_mat + sigma_n ** 2 * np.eye(size))
        self.k_mat = k_mat

    def get_kernel_vector(self, x: ChemicalEnvironment, d_1: int)-> np.ndarray:
        """ Compute kernel vector.

        :param x: data point to compare against kernel matrix
        :type x: ChemicalEnvironment
        :param d_1:
        :type d_1: int
        :return: kernel vector
        :rtype: np.ndarray
        """

        ds = [1, 2, 3]
        size = len(self.training_data)*3
        k_v = np.zeros(size,)

        for m_index in range(size):
            x_2 = self.training_data[int(math.floor(m_index/3))]
            d_2 = ds[m_index % 3]
            k_v[m_index] = self.kernel(x, x_2, d_1, d_2,
                                       self.sigma_f, self.length_scale)

        return k_v

    def set_alpha(self):
        """ Set weight vector alpha. """
        ts1 = solve_triangular(self.l_mat, self.training_labels_np, lower=True)
        self.alpha = solve_triangular(self.l_mat.transpose(), ts1)

def minus_like_hyp(hyp: List[float], gp: GaussianProcess,
                   verbose: bool = True) -> float:
    """Get minus likelihood as a function of hyperparameters.

    :param hyp: hyperparmeters to optimize: signal var, length scale, noise var
    :type hyp: [float, float, float]
    :param gp: gp object
    :type gp: GaussianProcess
    :param verbose: whether to print hyperparameters and likelihood
    :type verbose: bool
    :return: negative likelihood
    :rtype: float
    """
    like = gp.like_hyp(hyp)
    minus_like = -like

    if verbose:
        print(hyp)
        print(like)

    return minus_like
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

import numpy
import itertools
from .. import types
from .templates import (AttributeTemplate, AbstractTemplate,
                                    Registry, signature)

from ..numpy_support import (ufunc_find_matching_loop,
                             supported_ufunc_loop, as_dtype)

from ..typeinfer import TypingError

registry = Registry()
builtin = registry.register
builtin_global = registry.register_global
builtin_attr = registry.register_attr

@builtin_attr
class NumpyModuleAttribute(AttributeTemplate):
    # note: many unary ufuncs are added later on, using setattr
    key = types.Module(numpy)

class Numpy_rules_ufunc(AbstractTemplate):
    def generic(self, args, kws):
        ufunc = self.key
        nin = ufunc.nin
        nout = ufunc.nout
        nargs = ufunc.nargs

        # preconditions
        assert nargs == nin + nout

        if nout > 1:
            msg = "ufunc '{0}': not supported in this mode (more than 1 output)"
            raise TypingError(msg=msg.format(ufunc.__name__))

        if len(args) < nin:
            msg = "ufunc '{0}': not enough arguments ({1} found, {2} required)"
            raise TypingError(msg=msg.format(ufunc.__name__, len(args), nin))

        if len(args) > nargs:
            msg = "ufunc '{0}': too many arguments ({1} found, {2} maximum)"
            raise TypingError(msg=msg.format(ufunc.__name__, len(args), nargs))

        arg_ndims = [a.ndim if isinstance(a, types.Array) else 0 for a in args]
        ndims = max(arg_ndims)

        # explicit outputs must be arrays (no explicit scalar return values supported)
        explicit_outputs = args[nin:]

        # all the explicit outputs must match the number max number of dimensions
        if not all((d == ndims for d in arg_ndims[nin:])):
            msg = "ufunc '{0}' called with unsuitable explicit output arrays."
            raise TypingError(msg=msg.format(ufunc.__name__))

        if not all((isinstance(output, types.Array) for output in explicit_outputs)):
            msg = "ufunc '{0}' called with an explicit output that is not an array"
            raise TypingError(msg=msg.format(ufunc.__name__))

        # find the kernel to use, based only in the input types (as does NumPy)
        base_types = [x.dtype if isinstance(x, types.Array) else x for x in args]
        ufunc_loop = ufunc_find_matching_loop(ufunc, base_types)
        if ufunc_loop is None:
            raise TypingError("can't resolve ufunc {0} for types {1}".format(ufunc.__name__, args))

        # check if all the types involved in the ufunc loop are supported in this mode
        if not supported_ufunc_loop(ufunc, ufunc_loop):
            msg = "ufunc '{0}' using the loop '{1}' not supported in this mode"
            raise TypingError(msg=msg.format(ufunc.__name__, ufunc_loop.ufunc_sig))

        # if there is any explicit output type, check that it is valid
        explicit_outputs_np = [as_dtype(tp.dtype) for tp in explicit_outputs]

        # Numpy will happily use unsafe conversions (although it will actually warn)
        if not all (numpy.can_cast(fromty, toty, 'unsafe') for (fromty, toty) in
                    zip(ufunc_loop.numpy_outputs, explicit_outputs_np)):
            msg = "ufunc '{0}' can't cast result to explicit result type"
            raise TypingError(msg=msg.format(ufunc.__name__))

        # A valid loop was found that is compatible. The result of type inference should
        # be based on the explicit output types, and when not available with the type given
        # by the selected NumPy loop
        out = list(explicit_outputs)
        implicit_output_count = nout - len(explicit_outputs)
        if implicit_output_count > 0:
            # XXX this is currently wrong for datetime64 and timedelta64,
            # as ufunc_find_matching_loop() doesn't do any type inference.
            ret_tys = ufunc_loop.outputs[-implicit_output_count:]
            if ndims > 0:
                # XXX Not sure 'A' layout is correct...
                ret_tys = [types.Array(dtype=ret_ty, ndim=ndims, layout='A')
                           for ret_ty in ret_tys]
            out.extend(ret_tys)

        # note: although the previous code should support multiple return values, only one
        #       is supported as of now (signature may not support more than one).
        #       there is an check enforcing only one output
        out.extend(args)
        return signature(*out)

# list of unary ufuncs to register

_math_operations = [ "add", "subtract", "multiply",
                     "logaddexp", "logaddexp2", "true_divide",
                     "floor_divide", "negative", "power",
                     "remainder", "fmod", "absolute",
                     "rint", "sign", "conjugate", "exp", "exp2",
                     "log", "log2", "log10", "expm1", "log1p",
                     "sqrt", "square", "reciprocal",
                     "divide", "mod", "abs", "fabs" ]

_trigonometric_functions = [ "sin", "cos", "tan", "arcsin",
                             "arccos", "arctan", "arctan2",
                             "hypot", "sinh", "cosh", "tanh",
                             "arcsinh", "arccosh", "arctanh",
                             "deg2rad", "rad2deg", "degrees",
                             "radians" ]

_bit_twiddling_functions = ["bitwise_and", "bitwise_or",
                            "bitwise_xor", "invert",
                            "left_shift", "right_shift",
                            "bitwise_not" ]

_comparison_functions = [ "greater", "greater_equal", "less",
                          "less_equal", "not_equal", "equal",
                          "logical_and", "logical_or",
                          "logical_xor", "logical_not",
                          "maximum", "minimum", "fmax", "fmin" ]

_floating_functions = [ "isfinite", "isinf", "isnan", "signbit",
                        "copysign", "nextafter", "modf", "ldexp",
                        "frexp", "floor", "ceil", "trunc",
                        "spacing" ]

# This is a set of the ufuncs that are not yet supported by Lowering. In order
# to trigger no-python mode we must not register them until their Lowering is
# implemented.
#
# It also works as a nice TODO list for ufunc support :)
_unsupported = set([ numpy.frexp, # this one is tricky, as it has 2 returns
                     numpy.modf,  # this one also has 2 returns
                 ])

# a list of ufuncs that are in fact aliases of other ufuncs. They need to insert the
# resolve method, but not register the ufunc itself
_aliases = set(["bitwise_not", "mod", "abs"])

#in python3 numpy.divide is mapped to numpy.true_divide
if numpy.divide == numpy.true_divide:
    _aliases.add("divide")

def _numpy_ufunc(name):
    func = getattr(numpy, name)
    class typing_class(Numpy_rules_ufunc):
        key = func

    typing_class.__name__ = "resolve_{0}".format(name)
    # Add the resolve method to NumpyModuleAttribute
    setattr(NumpyModuleAttribute, typing_class.__name__,
            lambda s, m: types.Function(typing_class))

    if not name in _aliases:
        builtin_global(func, types.Function(typing_class))

all_ufuncs = sum([_math_operations, _trigonometric_functions,
                  _bit_twiddling_functions, _comparison_functions,
                  _floating_functions], [])

supported_ufuncs = [x for x in all_ufuncs if x not in _unsupported]

for func in supported_ufuncs:
    _numpy_ufunc(func)

all_ufuncs = [getattr(numpy, name) for name in all_ufuncs]
supported_ufuncs = [getattr(numpy, name) for name in supported_ufuncs]

del _math_operations, _trigonometric_functions, _bit_twiddling_functions
del _comparison_functions, _floating_functions, _unsupported
del _aliases, _numpy_ufunc

# -----------------------------------------------------------------------------
# Install global reduction functions

# Functions where input domain and output domain are the same
class Numpy_homogenous_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        return signature(arr.dtype, arr)

# Functions where domain and range are possibly different formats
class Numpy_expanded_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        if isinstance(arr.dtype, types.Integer):
            # Expand to a machine int, not larger (like Numpy)
            if arr.dtype.signed:
                return signature(max(arr.dtype, types.intp), arr)
            else:
                return signature(max(arr.dtype, types.uintp), arr)
        else:
            return signature(arr.dtype, arr)

class Numpy_heterogenous_reduction_real(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        if arr.dtype in types.integer_domain:
            return signature(types.float64, arr)
        else:
            return signature(arr.dtype, arr)

class Numpy_index_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        return signature(types.int64, arr)

# Function to glue attributes onto the numpy-esque object
def _numpy_reduction(fname, rClass):
    npyfn = getattr(numpy, fname)
    cls = type("Numpy_reduce_{0}".format(npyfn), (rClass,), dict(key=npyfn))
    semiBound = lambda self, mod: types.Function(cls)
    setattr(NumpyModuleAttribute, "resolve_{0}".format(fname), semiBound)

for func in ['min', 'max']:
    _numpy_reduction(func, Numpy_homogenous_reduction)

for func in ['sum', 'prod']:
    _numpy_reduction(func, Numpy_expanded_reduction)

for func in ['mean', 'var', 'std']:
    _numpy_reduction(func, Numpy_heterogenous_reduction_real)

for func in ['argmin', 'argmax']:
    _numpy_reduction(func, Numpy_index_reduction)

# -----------------------------------------------------------------------------
# Numpy scalar constructors

# Register numpy.int8, etc. as convertors to the equivalent Numba types
np_types = set(getattr(numpy, str(nb_type)) for nb_type in types.number_domain)
# Those may or may not be aliases (depending on the Numpy build / version)
np_types.add(numpy.intc)
np_types.add(numpy.intp)
np_types.add(numpy.uintc)
np_types.add(numpy.uintp)

def register_casters(register_global):
    for np_type in np_types:
        nb_type = getattr(types, np_type.__name__)

        class Caster(AbstractTemplate):
            key = np_type
            restype = nb_type

            def generic(self, args, kws):
                assert not kws
                [a] = args
                if a in types.number_domain:
                    return signature(self.restype, a)

        register_global(np_type, types.Function(Caster))

register_casters(builtin_global)

# -----------------------------------------------------------------------------
# Miscellaneous functions

@builtin
class NdEnumerate(AbstractTemplate):
    key = numpy.ndenumerate

    def generic(self, args, kws):
        assert not kws
        arr, = args

        if isinstance(arr, types.Array):
            enumerate_type = types.NumpyNdEnumerateType(arr)
            return signature(enumerate_type, *args)

builtin_global(numpy.ndenumerate, types.Function(NdEnumerate))

@builtin
class NdIndex(AbstractTemplate):
    key = numpy.ndindex

    def generic(self, args, kws):
        assert not kws

        # Either ndindex(shape) or ndindex(*shape)
        if len(args) == 1 and isinstance(args[0], types.UniTuple):
            shape = list(args[0])
        else:
            shape = args

        if shape and all(isinstance(x, types.Integer) for x in shape):
            iterator_type = types.NumpyNdIndexType(len(shape))
            return signature(iterator_type, *args)

builtin_global(numpy.ndindex, types.Function(NdIndex))

builtin_global(numpy, types.Module(numpy))
<commit_msg>Moved some input checking out of Numpy_rules_ufunc.generic() and into a class method for use by DUFunc's.<commit_after>from __future__ import absolute_import, print_function

import numpy
import itertools
from .. import types
from .templates import (AttributeTemplate, AbstractTemplate,
                                    Registry, signature)

from ..numpy_support import (ufunc_find_matching_loop,
                             supported_ufunc_loop, as_dtype)

from ..typeinfer import TypingError

registry = Registry()
builtin = registry.register
builtin_global = registry.register_global
builtin_attr = registry.register_attr

@builtin_attr
class NumpyModuleAttribute(AttributeTemplate):
    # note: many unary ufuncs are added later on, using setattr
    key = types.Module(numpy)

class Numpy_rules_ufunc(AbstractTemplate):
    @classmethod
    def _handle_inputs(cls, ufunc, args, kws):
        nin = ufunc.nin
        nout = ufunc.nout
        nargs = ufunc.nargs

        # preconditions
        assert nargs == nin + nout

        if nout > 1:
            msg = "ufunc '{0}': not supported in this mode (more than 1 output)"
            raise TypingError(msg=msg.format(ufunc.__name__))

        if len(args) < nin:
            msg = "ufunc '{0}': not enough arguments ({1} found, {2} required)"
            raise TypingError(msg=msg.format(ufunc.__name__, len(args), nin))

        if len(args) > nargs:
            msg = "ufunc '{0}': too many arguments ({1} found, {2} maximum)"
            raise TypingError(msg=msg.format(ufunc.__name__, len(args), nargs))

        arg_ndims = [a.ndim if isinstance(a, types.Array) else 0 for a in args]
        ndims = max(arg_ndims)

        # explicit outputs must be arrays (no explicit scalar return values supported)
        explicit_outputs = args[nin:]

        # all the explicit outputs must match the number max number of dimensions
        if not all((d == ndims for d in arg_ndims[nin:])):
            msg = "ufunc '{0}' called with unsuitable explicit output arrays."
            raise TypingError(msg=msg.format(ufunc.__name__))

        if not all((isinstance(output, types.Array) for output in explicit_outputs)):
            msg = "ufunc '{0}' called with an explicit output that is not an array"
            raise TypingError(msg=msg.format(ufunc.__name__))

        # find the kernel to use, based only in the input types (as does NumPy)
        base_types = [x.dtype if isinstance(x, types.Array) else x for x in args]
        return base_types, explicit_outputs, ndims

    def generic(self, args, kws):
        ufunc = self.key
        base_types, explicit_outputs, _ = self._handle_inputs(ufunc, args, kws)
        ufunc_loop = ufunc_find_matching_loop(ufunc, base_types)
        if ufunc_loop is None:
            raise TypingError("can't resolve ufunc {0} for types {1}".format(ufunc.__name__, args))

        # check if all the types involved in the ufunc loop are supported in this mode
        if not supported_ufunc_loop(ufunc, ufunc_loop):
            msg = "ufunc '{0}' using the loop '{1}' not supported in this mode"
            raise TypingError(msg=msg.format(ufunc.__name__, ufunc_loop.ufunc_sig))

        # if there is any explicit output type, check that it is valid
        explicit_outputs_np = [as_dtype(tp.dtype) for tp in explicit_outputs]

        # Numpy will happily use unsafe conversions (although it will actually warn)
        if not all (numpy.can_cast(fromty, toty, 'unsafe') for (fromty, toty) in
                    zip(ufunc_loop.numpy_outputs, explicit_outputs_np)):
            msg = "ufunc '{0}' can't cast result to explicit result type"
            raise TypingError(msg=msg.format(ufunc.__name__))

        # A valid loop was found that is compatible. The result of type inference should
        # be based on the explicit output types, and when not available with the type given
        # by the selected NumPy loop
        out = list(explicit_outputs)
        implicit_output_count = ufunc.nout - len(explicit_outputs)
        if implicit_output_count > 0:
            # XXX this is currently wrong for datetime64 and timedelta64,
            # as ufunc_find_matching_loop() doesn't do any type inference.
            ret_tys = ufunc_loop.outputs[-implicit_output_count:]
            if ndims > 0:
                # XXX Not sure 'A' layout is correct...
                ret_tys = [types.Array(dtype=ret_ty, ndim=ndims, layout='A')
                           for ret_ty in ret_tys]
            out.extend(ret_tys)

        # note: although the previous code should support multiple return values, only one
        #       is supported as of now (signature may not support more than one).
        #       there is an check enforcing only one output
        out.extend(args)
        return signature(*out)

# list of unary ufuncs to register

_math_operations = [ "add", "subtract", "multiply",
                     "logaddexp", "logaddexp2", "true_divide",
                     "floor_divide", "negative", "power",
                     "remainder", "fmod", "absolute",
                     "rint", "sign", "conjugate", "exp", "exp2",
                     "log", "log2", "log10", "expm1", "log1p",
                     "sqrt", "square", "reciprocal",
                     "divide", "mod", "abs", "fabs" ]

_trigonometric_functions = [ "sin", "cos", "tan", "arcsin",
                             "arccos", "arctan", "arctan2",
                             "hypot", "sinh", "cosh", "tanh",
                             "arcsinh", "arccosh", "arctanh",
                             "deg2rad", "rad2deg", "degrees",
                             "radians" ]

_bit_twiddling_functions = ["bitwise_and", "bitwise_or",
                            "bitwise_xor", "invert",
                            "left_shift", "right_shift",
                            "bitwise_not" ]

_comparison_functions = [ "greater", "greater_equal", "less",
                          "less_equal", "not_equal", "equal",
                          "logical_and", "logical_or",
                          "logical_xor", "logical_not",
                          "maximum", "minimum", "fmax", "fmin" ]

_floating_functions = [ "isfinite", "isinf", "isnan", "signbit",
                        "copysign", "nextafter", "modf", "ldexp",
                        "frexp", "floor", "ceil", "trunc",
                        "spacing" ]

# This is a set of the ufuncs that are not yet supported by Lowering. In order
# to trigger no-python mode we must not register them until their Lowering is
# implemented.
#
# It also works as a nice TODO list for ufunc support :)
_unsupported = set([ numpy.frexp, # this one is tricky, as it has 2 returns
                     numpy.modf,  # this one also has 2 returns
                 ])

# a list of ufuncs that are in fact aliases of other ufuncs. They need to insert the
# resolve method, but not register the ufunc itself
_aliases = set(["bitwise_not", "mod", "abs"])

#in python3 numpy.divide is mapped to numpy.true_divide
if numpy.divide == numpy.true_divide:
    _aliases.add("divide")

def _numpy_ufunc(name):
    func = getattr(numpy, name)
    class typing_class(Numpy_rules_ufunc):
        key = func

    typing_class.__name__ = "resolve_{0}".format(name)
    # Add the resolve method to NumpyModuleAttribute
    setattr(NumpyModuleAttribute, typing_class.__name__,
            lambda s, m: types.Function(typing_class))

    if not name in _aliases:
        builtin_global(func, types.Function(typing_class))

all_ufuncs = sum([_math_operations, _trigonometric_functions,
                  _bit_twiddling_functions, _comparison_functions,
                  _floating_functions], [])

supported_ufuncs = [x for x in all_ufuncs if x not in _unsupported]

for func in supported_ufuncs:
    _numpy_ufunc(func)

all_ufuncs = [getattr(numpy, name) for name in all_ufuncs]
supported_ufuncs = [getattr(numpy, name) for name in supported_ufuncs]

del _math_operations, _trigonometric_functions, _bit_twiddling_functions
del _comparison_functions, _floating_functions, _unsupported
del _aliases, _numpy_ufunc

# -----------------------------------------------------------------------------
# Install global reduction functions

# Functions where input domain and output domain are the same
class Numpy_homogenous_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        return signature(arr.dtype, arr)

# Functions where domain and range are possibly different formats
class Numpy_expanded_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        if isinstance(arr.dtype, types.Integer):
            # Expand to a machine int, not larger (like Numpy)
            if arr.dtype.signed:
                return signature(max(arr.dtype, types.intp), arr)
            else:
                return signature(max(arr.dtype, types.uintp), arr)
        else:
            return signature(arr.dtype, arr)

class Numpy_heterogenous_reduction_real(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        if arr.dtype in types.integer_domain:
            return signature(types.float64, arr)
        else:
            return signature(arr.dtype, arr)

class Numpy_index_reduction(AbstractTemplate):
    def generic(self, args, kws):
        assert not kws
        [arr] = args
        return signature(types.int64, arr)

# Function to glue attributes onto the numpy-esque object
def _numpy_reduction(fname, rClass):
    npyfn = getattr(numpy, fname)
    cls = type("Numpy_reduce_{0}".format(npyfn), (rClass,), dict(key=npyfn))
    semiBound = lambda self, mod: types.Function(cls)
    setattr(NumpyModuleAttribute, "resolve_{0}".format(fname), semiBound)

for func in ['min', 'max']:
    _numpy_reduction(func, Numpy_homogenous_reduction)

for func in ['sum', 'prod']:
    _numpy_reduction(func, Numpy_expanded_reduction)

for func in ['mean', 'var', 'std']:
    _numpy_reduction(func, Numpy_heterogenous_reduction_real)

for func in ['argmin', 'argmax']:
    _numpy_reduction(func, Numpy_index_reduction)

# -----------------------------------------------------------------------------
# Numpy scalar constructors

# Register numpy.int8, etc. as convertors to the equivalent Numba types
np_types = set(getattr(numpy, str(nb_type)) for nb_type in types.number_domain)
# Those may or may not be aliases (depending on the Numpy build / version)
np_types.add(numpy.intc)
np_types.add(numpy.intp)
np_types.add(numpy.uintc)
np_types.add(numpy.uintp)

def register_casters(register_global):
    for np_type in np_types:
        nb_type = getattr(types, np_type.__name__)

        class Caster(AbstractTemplate):
            key = np_type
            restype = nb_type

            def generic(self, args, kws):
                assert not kws
                [a] = args
                if a in types.number_domain:
                    return signature(self.restype, a)

        register_global(np_type, types.Function(Caster))

register_casters(builtin_global)

# -----------------------------------------------------------------------------
# Miscellaneous functions

@builtin
class NdEnumerate(AbstractTemplate):
    key = numpy.ndenumerate

    def generic(self, args, kws):
        assert not kws
        arr, = args

        if isinstance(arr, types.Array):
            enumerate_type = types.NumpyNdEnumerateType(arr)
            return signature(enumerate_type, *args)

builtin_global(numpy.ndenumerate, types.Function(NdEnumerate))

@builtin
class NdIndex(AbstractTemplate):
    key = numpy.ndindex

    def generic(self, args, kws):
        assert not kws

        # Either ndindex(shape) or ndindex(*shape)
        if len(args) == 1 and isinstance(args[0], types.UniTuple):
            shape = list(args[0])
        else:
            shape = args

        if shape and all(isinstance(x, types.Integer) for x in shape):
            iterator_type = types.NumpyNdIndexType(len(shape))
            return signature(iterator_type, *args)

builtin_global(numpy.ndindex, types.Function(NdIndex))

builtin_global(numpy, types.Module(numpy))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
################################################################################
#        _____ _                  _____           _                            #
#       / ____(_)                / ____|         | |                           #
#      | |     _ ___  ___ ___   | (___  _   _ ___| |_ ___ _ __ ___  ___        #
#      | |    | / __|/ __/ _ \   \___ \| | | / __| __/ _ \ '_ ` _ \/ __|       #
#      | |____| \__ \ (_| (_) |  ____) | |_| \__ \ ||  __/ | | | | \__ \       #
#       \_____|_|___/\___\___/  |_____/ \__, |___/\__\___|_| |_| |_|___/       #
#                                        __/ |                                 #
#                                       |___/                                  #
#           _  __                       _____       _  _____ ______            #
#          | |/ /                      / ____|     | |/ ____|  ____|           #
#          | ' / ___  _ __ ___  __ _  | (___   ___ | | (___ | |__              #
#          |  < / _ \| '__/ _ \/ _` |  \___ \ / _ \| |\___ \|  __|             #
#          | . \ (_) | | |  __/ (_| |  ____) | (_) | |____) | |____            #
#          |_|\_\___/|_|  \___|\__,_| |_____/ \___/|_|_____/|______|           #
#                                                                              #
################################################################################
#                                                                              #
# Copyright (c) 2016 Cisco Systems                                             #
# All Rights Reserved.                                                         #
#                                                                              #
# Licensed under the Apache License, Version 2.0 (the "License"); you may      #
# not use this file except in compliance with the License. You may obtain      #
# a copy of the License at                                                     #
#                                                                              #
# http://www.apache.org/licenses/LICENSE-2.0                                   #
#                                                                              #
# Unless required by applicable law or agreed to in writing, software          #
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT #
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the  #
#    License for the specific language governing permissions and limitations   #
#    under the License.                                                        #
#                                                                              #
################################################################################

from archon import *
from manager import Manager

from show import *
#===============================================================================
# Create your views here.
#===============================================================================

@pageview(Manager)
def device(Req, Man, View):
    if Req.Method == 'GET':
        plen = len(Req.Path)
        if plen > 4: return device_one(Req, Man, View)
        elif plen > 3: return device_all(Req, Man, View)
        else: return device_all(Req, Man, View)

@pageview(Manager)
def tenant(Req, Man, View):
    if Req.Method == 'GET':
        if len(Req.Path) > 3: return tenant_one(Req, Man, View)
        else: return tenant_all(Req, Man, View)

@pageview(Manager)
def epg(Req, Man, View):

    key = ['name', 'childAction', 'configIssues', 'configSt', 'descr', 'dn', 'fwdCtrl', 'isAttrBasedEPg', 'lcOwn', 'matchT', 'modTs', 'monPolDn', 'pcEnfPref', 'pcTag', 'prefGrMemb', 'prio', 'scope', 'status', 'triggerSt', 'txId', 'uid']

    return 'Endpoint Group'

@pageview(Manager)
def ep(Req, Man, View):
    return 'Endpoint'

@pageview(Manager)
def contract(Req, Man, View):
    return 'Contract'

@pageview(Manager)
def external(Req, Man, View):
    return 'External Network'

@pageview(Manager)
def fault(Req, Man, View):
    return 'Fault'

@pageview(Manager)
def intf_util(Req, Man, View):
    return 'Interface Utilization'

@pageview(Manager)
def epg_util(Req, Man, View):
    return 'Epg Utilization'

@pageview(Manager)
def eptracker(Req, Man, View):
    return 'EP Tracker'

@pageview(Manager)
def ofinder(Req, Man, View):
    return 'Object Finder'

@pageview(Manager)
def config(Req, Man, View):

    alert = None

    if Req.Method == 'POST':
        if Man.addDomain(Req.Data['domain_name'], Req.Data['ip'], Req.Data['user'], Req.Data['passwd']):
            alert = Alert(View('Connection Success'),
                          View('Setting %s with %s') % (Req.Data['ip'], Req.Data['domain_name']),
                          **{'class' : 'alert-success'})
        else:
            alert = Alert(View('Connection Failed'),
                          View('Incorrect Setting %s') % Req.Data['domain_name'],
                          **{'class' : 'alert-danger'})

    elif Req.Method == 'DELETE':
        if Man.delDomain(Req.Path[2]):
            alert = Alert(View('Disconnection Success'),
                          View('Erasing %s') % Req.Path[2],
                          **{'class' : 'alert-success'})
        else:
            alert = Alert(View('Disconnection Failed'),
                          View('Incorrect Erasing %s') % Req.Path[2],
                          **{'class' : 'alert-danger'})

    View.Menu.html(
        Modal(View('Register APIC Domain'), BUTTON(**{'class' : 'btn-primary', 'style' : 'float:right;'}).html(View('Connect APIC'))).html(
            Post('/aci/conf', View('Register'), **{'class' : 'btn-primary', 'style' : 'float:right;'}
            ).Text('domain_name', Post.TopLabel(View('APIC Domain Name')), placeholder=View('Unique Name Required')
            ).Text('ip', Post.TopLabel(View('APIC Address')), placeholder='XXX.XXX.XXX.XXX'
            ).Text('user', Post.TopLabel(View('Admin ID')), placeholder='admin'
            ).Password(label=Post.TopLabel(View('Password')))
        )
    )

    table = FooTable(View('Domain Name'), View('+APIC IP'), View('+Administrator ID'), View('+Start Connections'), View('+Max Connections'), '')

    for domain_name in Man:
        table.record(domain_name,
                     Man[domain_name]['ip'],
                     Man[domain_name]['user'],
                     Man[domain_name]['conns'],
                     Man[domain_name]['conn_max'],
                     DelButton('/aci/conf/' + domain_name, View('Delete'), tail=True))

    if alert != None: View.Page.html(alert)
    View.Page.html(table)

<commit_msg>Arranged<commit_after># -*- coding: utf-8 -*-
################################################################################
#        _____ _                  _____           _                            #
#       / ____(_)                / ____|         | |                           #
#      | |     _ ___  ___ ___   | (___  _   _ ___| |_ ___ _ __ ___  ___        #
#      | |    | / __|/ __/ _ \   \___ \| | | / __| __/ _ \ '_ ` _ \/ __|       #
#      | |____| \__ \ (_| (_) |  ____) | |_| \__ \ ||  __/ | | | | \__ \       #
#       \_____|_|___/\___\___/  |_____/ \__, |___/\__\___|_| |_| |_|___/       #
#                                        __/ |                                 #
#                                       |___/                                  #
#           _  __                       _____       _  _____ ______            #
#          | |/ /                      / ____|     | |/ ____|  ____|           #
#          | ' / ___  _ __ ___  __ _  | (___   ___ | | (___ | |__              #
#          |  < / _ \| '__/ _ \/ _` |  \___ \ / _ \| |\___ \|  __|             #
#          | . \ (_) | | |  __/ (_| |  ____) | (_) | |____) | |____            #
#          |_|\_\___/|_|  \___|\__,_| |_____/ \___/|_|_____/|______|           #
#                                                                              #
################################################################################
#                                                                              #
# Copyright (c) 2016 Cisco Systems                                             #
# All Rights Reserved.                                                         #
#                                                                              #
# Licensed under the Apache License, Version 2.0 (the "License"); you may      #
# not use this file except in compliance with the License. You may obtain      #
# a copy of the License at                                                     #
#                                                                              #
# http://www.apache.org/licenses/LICENSE-2.0                                   #
#                                                                              #
# Unless required by applicable law or agreed to in writing, software          #
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT #
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the  #
#    License for the specific language governing permissions and limitations   #
#    under the License.                                                        #
#                                                                              #
################################################################################

from archon import *
from manager import Manager

from show import *
#===============================================================================
# Create your views here.
#===============================================================================

@pageview(Manager)
def device(Req, Man, View):
    if Req.Method == 'GET':
        plen = len(Req.Path)
        if plen > 4: return device_one(Req, Man, View)
        elif plen > 3: return device_all(Req, Man, View)
        else: return device_all(Req, Man, View)

@pageview(Manager)
def tenant(Req, Man, View):
    if Req.Method == 'GET':
        if len(Req.Path) > 3: return tenant_one(Req, Man, View)
        else: return tenant_all(Req, Man, View)

@pageview(Manager)
def epg(Req, Man, View):
    return 'Endpoint Group'

@pageview(Manager)
def ep(Req, Man, View):
    return 'Endpoint'

@pageview(Manager)
def contract(Req, Man, View):
    return 'Contract'

@pageview(Manager)
def external(Req, Man, View):
    return 'External Network'

@pageview(Manager)
def fault(Req, Man, View):
    return 'Fault'

@pageview(Manager)
def intf_util(Req, Man, View):
    return 'Interface Utilization'

@pageview(Manager)
def epg_util(Req, Man, View):
    return 'Epg Utilization'

@pageview(Manager)
def eptracker(Req, Man, View):
    return 'EP Tracker'

@pageview(Manager)
def ofinder(Req, Man, View):
    return 'Object Finder'

@pageview(Manager)
def config(Req, Man, View):

    alert = None

    if Req.Method == 'POST':
        if Man.addDomain(Req.Data['domain_name'], Req.Data['ip'], Req.Data['user'], Req.Data['passwd']):
            alert = Alert(View('Connection Success'),
                          View('Setting %s with %s') % (Req.Data['ip'], Req.Data['domain_name']),
                          **{'class' : 'alert-success'})
        else:
            alert = Alert(View('Connection Failed'),
                          View('Incorrect Setting %s') % Req.Data['domain_name'],
                          **{'class' : 'alert-danger'})

    elif Req.Method == 'DELETE':
        if Man.delDomain(Req.Path[2]):
            alert = Alert(View('Disconnection Success'),
                          View('Erasing %s') % Req.Path[2],
                          **{'class' : 'alert-success'})
        else:
            alert = Alert(View('Disconnection Failed'),
                          View('Incorrect Erasing %s') % Req.Path[2],
                          **{'class' : 'alert-danger'})

    View.Menu.html(
        Modal(View('Register APIC Domain'), BUTTON(**{'class' : 'btn-primary', 'style' : 'float:right;'}).html(View('Connect APIC'))).html(
            Post('/aci/conf', View('Register'), **{'class' : 'btn-primary', 'style' : 'float:right;'}
            ).Text('domain_name', Post.TopLabel(View('APIC Domain Name')), placeholder=View('Unique Name Required')
            ).Text('ip', Post.TopLabel(View('APIC Address')), placeholder='XXX.XXX.XXX.XXX'
            ).Text('user', Post.TopLabel(View('Admin ID')), placeholder='admin'
            ).Password(label=Post.TopLabel(View('Password')))
        )
    )

    table = FooTable(View('Domain Name'), View('+APIC IP'), View('+Administrator ID'), View('+Start Connections'), View('+Max Connections'), '')

    for domain_name in Man:
        table.record(domain_name,
                     Man[domain_name]['ip'],
                     Man[domain_name]['user'],
                     Man[domain_name]['conns'],
                     Man[domain_name]['conn_max'],
                     DelButton('/aci/conf/' + domain_name, View('Delete'), tail=True))

    if alert != None: View.Page.html(alert)
    View.Page.html(table)

<|endoftext|>


<commit_before>import httplib2
import json
import itertools
from collections import defaultdict

from apiclient.discovery import build
from oauth2client.file import Storage
from oauth2client.client import flow_from_clientsecrets
from oauth2client.tools import run

class Locations(object):
    CLIENT_SECRETS_FILE = 'client_secrets.json'

    FLOW = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
        scope = "https://www.googleapis.com/auth/analytics.readonly",
        message = "Something seems to have gone wrong, check the client secrets file")

    def __init__(self):
        storage = Storage("tokens.dat")
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run(FLOW, storage)

        http = httplib2.Http()
        http = credentials.authorize(http)

        service = build("analytics","v3", http = http)

        query = service.data().ga().get(
            metrics = "ga:visits",
            dimensions = "ga:pagePath,ga:country",
            max_results = "5000",
            start_date = "2013-01-01",
            end_date = "2013-02-01",
            ids = "ga:63654109",
            filters = "ga:pagePath=~^/apply-for-a-licence/.*/form$")

        response = query.execute()['rows']

        self.results = response

<commit_msg>Fix referencing of oauth flow.<commit_after>import httplib2
import json
import itertools
from collections import defaultdict

from apiclient.discovery import build
from oauth2client.file import Storage
from oauth2client.client import flow_from_clientsecrets
from oauth2client.tools import run

class Locations(object):
    CLIENT_SECRETS_FILE = 'client_secrets.json'

    FLOW = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
        scope = "https://www.googleapis.com/auth/analytics.readonly",
        message = "Something seems to have gone wrong, check the client secrets file")

    def __init__(self):
        storage = Storage("tokens.dat")
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run(self.FLOW, storage)

        http = httplib2.Http()
        http = credentials.authorize(http)

        service = build("analytics","v3", http = http)

        query = service.data().ga().get(
            metrics = "ga:visits",
            dimensions = "ga:pagePath,ga:country",
            max_results = "5000",
            start_date = "2013-01-01",
            end_date = "2013-02-01",
            ids = "ga:63654109",
            filters = "ga:pagePath=~^/apply-for-a-licence/.*/form$")

        response = query.execute()['rows']

        self.results = response

<|endoftext|>