<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" Launch multiple buildbot slave instances on a single machine.  This script
is intended to be run at boot time. """

import multiprocessing
import os
import shutil
import socket
import subprocess
import sys
import time

buildbot_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             os.pardir))
sys.path.append(os.path.join(buildbot_path, 'master'))
sys.path.append(os.path.join(buildbot_path, 'site_config'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'scripts'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'site_config'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'buildbot_8_4p1'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'jinja2'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'twisted_8_1'))

import config
import slave_hosts_cfg
import slaves_cfg

SKIA_URL = 'https://skia.googlesource.com/buildbot.git'
GCLIENT = 'gclient.bat' if os.name == 'nt' else 'gclient'
GIT = 'git.bat' if os.name == 'nt' else 'git'

# How often we should check each buildslave's keepalive conditions, in seconds.
DEFAULT_POLL_INTERVAL = 60
PID_FILE = os.path.join('buildbot', 'third_party', 'chromium_buildbot', 'slave',
                        'twistd.pid')
# Maximum time (in seconds) to wait for PID_FILE to be written after the slave
# is launched.  If PID_FILE is not written by then, we assume an error occurred.
PID_TIMEOUT = 60.0

logger = None

# TODO(borenet): Share this code with launch_master.py.
def IsRunning(pid):
  """ Determine whether a process with the given PID is running.

  pid: string; the PID to test. If pid is None, return False.
  """
  if pid is None:
    return False
  if os.name == 'nt':
    cmd = ['tasklist', '/FI', '"PID eq %s"' % pid]
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    if proc.wait() != 0:
      raise Exception('Unable to poll process with PID %s' % pid)
    is_running = pid in proc.communicate()[0]
  else:
    cmd = ['cat', '/proc/%s/stat' % pid]
    is_running = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT).wait() == 0
  return is_running

class BuildSlaveManager(multiprocessing.Process):
  """ Manager process for BuildSlaves. Periodically checks that any
  keepalive_conditions are met and kills or starts the slave accordingly. """

  def __init__(self, slavename, slave_dir, copies, copy_src_dir, master_name,
               keepalive_conditions, poll_interval):
    """ Construct the BuildSlaveManager.

    slavename: string; the name of the slave to start.
    slave_dir: string; the directory in which to launch the slave.
    copies: list of dictionaries; files to copy into the slave's source
        checkout.
    copy_src_dir: string; directory in which the files to copy reside.
    master_name: string; name of the master to which this build slave connects.
        This is NOT the hostname of the master, which is obtained from the
        master class in config_private.py.
    keepalive_conditions: list; commands which must succeed in order for the
        slave to stay alive.
    poll_interval: number; how often to verify the keepalive_conditions, in
        seconds.
    """
    self._slavename = slavename
    self._slave_dir = slave_dir
    self._copies = copies
    self._copy_src_dir = os.path.abspath(copy_src_dir)
    self._keepalive_conditions = keepalive_conditions
    self._poll_interval = poll_interval
    self._master_name = master_name
    multiprocessing.Process.__init__(self)

  def _SyncSources(self):
    """ Run 'gclient sync' on the buildbot sources. """
    # Check out or update the buildbot scripts
    if os.name == 'nt':
      gclient = 'gclient.bat'
    else:
      gclient = 'gclient'
    proc = subprocess.Popen([gclient, 'config', SKIA_URL])
    if proc.wait() != 0:
      raise Exception('Could not successfully configure gclient.')
    proc = subprocess.Popen([gclient, 'sync', '-j1', '--force'])
    if proc.wait() != 0:
      raise Exception('Sync failed.')

    # Perform Copies
    for copy in self._copies:
      src = os.path.join(self._copy_src_dir, os.path.normpath(copy['source']))
      dest = os.path.normpath(copy['destination'])
      print 'Copying %s to %s' % (src, dest)
      shutil.copy(src, dest)

  def _LaunchSlave(self):
    """ Launch the BuildSlave. """
    if self._IsRunning():
      self._KillSlave()

    self._SyncSources()

    # Find the hostname of the master we're connecting to.
    master_host = config.Master.get(self._master_name).master_host

    os.chdir(os.path.join('buildbot', 'slave'))
    if os.name == 'nt':
      # We run different commands for the Windows shell
      cmd = 'setlocal&&'
      cmd += 'set TESTING_SLAVENAME=%s&&' % self._slavename
      cmd += 'set TESTING_MASTER=%s&&' % self._master_name
      cmd += 'set TESTING_MASTER_HOST=%s&&' % master_host
      cmd += 'run_slave.bat'
      cmd += '&& endlocal'
    else:
      proc = subprocess.Popen(['make', 'stop'])
      proc.wait()
      cmd = 'TESTING_SLAVENAME=%s ' % self._slavename
      cmd += 'TESTING_MASTER=%s ' % self._master_name
      cmd += 'TESTING_MASTER_HOST=%s ' % master_host
      cmd += 'make start'
    print 'Running cmd: %s' % cmd
    subprocess.Popen(cmd, shell=True)
    os.chdir(self._slave_dir)

    start_time = time.time()
    while not self._IsRunning():
      if time.time() - start_time > PID_TIMEOUT:
        raise Exception('Failed to launch %s' % self._slavename)
      time.sleep(1)

  def _IsRunning(self):
    """ Determine if this BuildSlave is running. If so, return its PID,
    otherwise, return None. """
    if os.path.isfile(PID_FILE):
      pid_file = open(PID_FILE)
      pid = str(pid_file.read()).rstrip()
      pid_file.close()
      if IsRunning(pid):
        return pid
    return None

  def _KillSlave(self):
    """ Kill the BuildSlave. """
    pid = self._IsRunning()
    if not pid:
      print 'BuildSlaveManager._KillSlave: Slave not running.'
      return
    os.chdir(os.path.join('buildbot', 'slave'))
    if os.name == 'nt':
      cmd = ['taskkill', '/F', '/T', '/PID', str(pid)]
    else:
      cmd = ['make', 'stop']
    if subprocess.Popen(cmd).wait() != 0:
      raise Exception('Failed to kill slave with pid %s' % str(pid))
    os.chdir(self._slave_dir)

  def run(self):
    """ Run the BuildSlaveManager. This overrides multiprocessing.Process's
    run() method. """
    os.chdir(self._slave_dir)
    self._SyncSources()
    self._slave_dir = os.path.abspath(os.curdir)
    if self._IsRunning():
      self._KillSlave()
    while True:
      print 'Checking keepalive conditions for %s' % self._slavename
      slave_can_run = True
      for keepalive_condition in self._keepalive_conditions:
        print 'Executing keepalive condition: %s' % keepalive_condition
        proc = subprocess.Popen(keepalive_condition, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        if proc.wait() != 0:
          print 'Keepalive condition failed for %s: %s' % (self._slavename,
                                                           keepalive_condition)
          print proc.communicate()[0]
          slave_can_run = False
          break
        print proc.communicate()[0]
      if not slave_can_run and self._IsRunning():
        self._KillSlave()
      elif slave_can_run and not self._IsRunning():
        self._LaunchSlave()
        print 'Successfully launched slave %s.' % self._slavename
      time.sleep(self._poll_interval)
    print 'Slave process for %s has finished.' % self._slavename

def RunSlave(slavename, copies):
  """ Launch a single slave, checking out the buildbot tree if necessary.

  slavename: string indicating the hostname of the build slave to launch.
  copies: dictionary with 'source' and 'destination' keys whose values are the
      current location and destination location within the buildbot checkout of
      files to be copied.
  """
  print 'Starting slave: %s' % slavename
  start_dir = os.path.realpath(os.curdir)
  slave_dir = os.path.join(start_dir, slavename)

  # Create the slave directory if needed
  if not os.path.isdir(slave_dir):
    print 'Creating directory: %s' % slave_dir
    os.makedirs(slave_dir)

  # Find the slave config dict for this slave.
  slave_cfg = {}
  for cfg in slaves_cfg.SLAVES:
    if cfg['hostname'] == slavename:
      slave_cfg = cfg
      break
  if not slave_cfg:
    raise Exception('No buildslave config found for %s!' % slavename)

  manager = BuildSlaveManager(slavename, slave_dir, copies, os.pardir,
                              slave_cfg['master'],
                              slave_cfg.get('keepalive_conditions', []),
                              DEFAULT_POLL_INTERVAL)
  manager.start()

class FileLogger:
  """ Write stdout to a log file. """
  def __init__(self, log_file_name):
    # Open the log file.
    self._logfile = open(log_file_name, 'w')
    self._stdout = sys.stdout
    sys.stdout = self

  def __del__(self):
    self._logfile.close()
    sys.stdout = self._stdout

  def fileno(self):
    return self._stdout.fileno()

  def write(self, data):
    self._logfile.write(data)
    self._stdout.write(data)
    # Always flush the log file.
    self._logfile.flush()

  def flush(self):
    self._logfile.flush()
    self._stdout.flush()

def ParseArgs(argv):
  """ Parse and validate command-line arguments. """

  class CollectedArgs(object):
    pass

  usage = (
"""launch_slaves.py: Launch build slaves.
python launch_slaves.py

-h, --help          Show this message.
""")

  def Exit(error_msg=None):
    if error_msg:
      print error_msg
    print usage
    sys.exit(1)

  while argv:
    arg = argv.pop(0)
    if arg == '-h' or arg == '--help':
      Exit()
    else:
      Exit('Unknown argument: %s' % arg)
  return CollectedArgs()

def main():
  """ Launch local build slave instances """
  # Gather command-line arguments.
  ParseArgs(sys.argv[1:])

  # Sync the buildbot code.
  subprocess.check_call([GCLIENT, 'sync', '--force', '-j1'])

  # Obtain configuration information about this build slave host machine.
  slave_host = slave_hosts_cfg.GetSlaveHostConfig(socket.gethostname())
  slaves = slave_host['slaves']
  copies = slave_host['copies']
  print 'Attempting to launch build slaves:'
  for slavename, _ in slaves:
    print '  %s' % slavename

  # Launch the build slaves
  for slavename, _ in slaves:
    RunSlave(slavename, copies)

if '__main__' == __name__:
  # Pipe all output to a log file.
  logfile = 'launch_slaves.log'
  if os.path.isfile(logfile):
    num = 1
    new_filename = logfile + '.' + str(num)
    while os.path.isfile(new_filename):
      num += 1
      new_filename = logfile + '.' + str(num)
    os.rename(logfile, new_filename)
  logger = FileLogger(logfile)
  sys.exit(main())
<commit_msg>Fix launch_slaves.py<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" Launch multiple buildbot slave instances on a single machine.  This script
is intended to be run at boot time. """

import multiprocessing
import os
import shutil
import socket
import subprocess
import sys
import time

buildbot_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             os.pardir))
sys.path.append(os.path.join(buildbot_path, 'master'))
sys.path.append(os.path.join(buildbot_path, 'site_config'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'scripts'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'site_config'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'buildbot_8_4p1'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'jinja2'))
sys.path.append(os.path.join(buildbot_path, 'third_party', 'chromium_buildbot',
                             'third_party', 'twisted_8_1'))

import config
import slave_hosts_cfg
import slaves_cfg

SKIA_URL = 'https://skia.googlesource.com/buildbot.git'
GCLIENT = 'gclient.bat' if os.name == 'nt' else 'gclient'
GIT = 'git.bat' if os.name == 'nt' else 'git'

# How often we should check each buildslave's keepalive conditions, in seconds.
DEFAULT_POLL_INTERVAL = 60
PID_FILE = os.path.join('buildbot', 'third_party', 'chromium_buildbot', 'slave',
                        'twistd.pid')
# Maximum time (in seconds) to wait for PID_FILE to be written after the slave
# is launched.  If PID_FILE is not written by then, we assume an error occurred.
PID_TIMEOUT = 60.0

logger = None

# TODO(borenet): Share this code with launch_master.py.
def IsRunning(pid):
  """ Determine whether a process with the given PID is running.

  pid: string; the PID to test. If pid is None, return False.
  """
  if pid is None:
    return False
  if os.name == 'nt':
    cmd = ['tasklist', '/FI', '"PID eq %s"' % pid]
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    if proc.wait() != 0:
      raise Exception('Unable to poll process with PID %s' % pid)
    is_running = pid in proc.communicate()[0]
  else:
    cmd = ['cat', '/proc/%s/stat' % pid]
    is_running = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT).wait() == 0
  return is_running

class BuildSlaveManager(multiprocessing.Process):
  """ Manager process for BuildSlaves. Periodically checks that any
  keepalive_conditions are met and kills or starts the slave accordingly. """

  def __init__(self, slavename, slave_dir, copies, copy_src_dir, master_name,
               keepalive_conditions, poll_interval):
    """ Construct the BuildSlaveManager.

    slavename: string; the name of the slave to start.
    slave_dir: string; the directory in which to launch the slave.
    copies: list of dictionaries; files to copy into the slave's source
        checkout.
    copy_src_dir: string; directory in which the files to copy reside.
    master_name: string; name of the master to which this build slave connects.
        This is NOT the hostname of the master, which is obtained from the
        master class in config_private.py.
    keepalive_conditions: list; commands which must succeed in order for the
        slave to stay alive.
    poll_interval: number; how often to verify the keepalive_conditions, in
        seconds.
    """
    self._slavename = slavename
    self._slave_dir = slave_dir
    self._copies = copies
    self._copy_src_dir = os.path.abspath(copy_src_dir)
    self._keepalive_conditions = keepalive_conditions
    self._poll_interval = poll_interval
    self._master_name = master_name
    multiprocessing.Process.__init__(self)

  def _SyncSources(self):
    """ Run 'gclient sync' on the buildbot sources. """
    # Check out or update the buildbot scripts
    if os.name == 'nt':
      gclient = 'gclient.bat'
    else:
      gclient = 'gclient'
    proc = subprocess.Popen([gclient, 'config', SKIA_URL])
    if proc.wait() != 0:
      raise Exception('Could not successfully configure gclient.')
    proc = subprocess.Popen([gclient, 'sync', '-j1', '--force'])
    if proc.wait() != 0:
      raise Exception('Sync failed.')

    # Perform Copies
    for copy in self._copies:
      src = os.path.join(self._copy_src_dir, os.path.normpath(copy['source']))
      dest = os.path.normpath(copy['destination'])
      print 'Copying %s to %s' % (src, dest)
      shutil.copy(src, dest)

  def _LaunchSlave(self):
    """ Launch the BuildSlave. """
    if self._IsRunning():
      self._KillSlave()

    self._SyncSources()

    # Find the hostname of the master we're connecting to.
    master_host = config.Master.set_active_master(self._master_name).master_host

    os.chdir(os.path.join('buildbot', 'slave'))
    if os.name == 'nt':
      # We run different commands for the Windows shell
      cmd = 'setlocal&&'
      cmd += 'set TESTING_SLAVENAME=%s&&' % self._slavename
      cmd += 'set TESTING_MASTER=%s&&' % self._master_name
      cmd += 'set TESTING_MASTER_HOST=%s&&' % master_host
      cmd += 'run_slave.bat'
      cmd += '&& endlocal'
    else:
      proc = subprocess.Popen(['make', 'stop'])
      proc.wait()
      cmd = 'TESTING_SLAVENAME=%s ' % self._slavename
      cmd += 'TESTING_MASTER=%s ' % self._master_name
      cmd += 'TESTING_MASTER_HOST=%s ' % master_host
      cmd += 'make start'
    print 'Running cmd: %s' % cmd
    subprocess.Popen(cmd, shell=True)
    os.chdir(self._slave_dir)

    start_time = time.time()
    while not self._IsRunning():
      if time.time() - start_time > PID_TIMEOUT:
        raise Exception('Failed to launch %s' % self._slavename)
      time.sleep(1)

  def _IsRunning(self):
    """ Determine if this BuildSlave is running. If so, return its PID,
    otherwise, return None. """
    if os.path.isfile(PID_FILE):
      pid_file = open(PID_FILE)
      pid = str(pid_file.read()).rstrip()
      pid_file.close()
      if IsRunning(pid):
        return pid
    return None

  def _KillSlave(self):
    """ Kill the BuildSlave. """
    pid = self._IsRunning()
    if not pid:
      print 'BuildSlaveManager._KillSlave: Slave not running.'
      return
    os.chdir(os.path.join('buildbot', 'slave'))
    if os.name == 'nt':
      cmd = ['taskkill', '/F', '/T', '/PID', str(pid)]
    else:
      cmd = ['make', 'stop']
    if subprocess.Popen(cmd).wait() != 0:
      raise Exception('Failed to kill slave with pid %s' % str(pid))
    os.chdir(self._slave_dir)

  def run(self):
    """ Run the BuildSlaveManager. This overrides multiprocessing.Process's
    run() method. """
    os.chdir(self._slave_dir)
    self._SyncSources()
    self._slave_dir = os.path.abspath(os.curdir)
    if self._IsRunning():
      self._KillSlave()
    while True:
      print 'Checking keepalive conditions for %s' % self._slavename
      slave_can_run = True
      for keepalive_condition in self._keepalive_conditions:
        print 'Executing keepalive condition: %s' % keepalive_condition
        proc = subprocess.Popen(keepalive_condition, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        if proc.wait() != 0:
          print 'Keepalive condition failed for %s: %s' % (self._slavename,
                                                           keepalive_condition)
          print proc.communicate()[0]
          slave_can_run = False
          break
        print proc.communicate()[0]
      if not slave_can_run and self._IsRunning():
        self._KillSlave()
      elif slave_can_run and not self._IsRunning():
        self._LaunchSlave()
        print 'Successfully launched slave %s.' % self._slavename
      time.sleep(self._poll_interval)
    print 'Slave process for %s has finished.' % self._slavename

def RunSlave(slavename, copies):
  """ Launch a single slave, checking out the buildbot tree if necessary.

  slavename: string indicating the hostname of the build slave to launch.
  copies: dictionary with 'source' and 'destination' keys whose values are the
      current location and destination location within the buildbot checkout of
      files to be copied.
  """
  print 'Starting slave: %s' % slavename
  start_dir = os.path.realpath(os.curdir)
  slave_dir = os.path.join(start_dir, slavename)

  # Create the slave directory if needed
  if not os.path.isdir(slave_dir):
    print 'Creating directory: %s' % slave_dir
    os.makedirs(slave_dir)

  # Find the slave config dict for this slave.
  slave_cfg = {}
  for cfg in slaves_cfg.SLAVES:
    if cfg['hostname'] == slavename:
      slave_cfg = cfg
      break
  if not slave_cfg:
    raise Exception('No buildslave config found for %s!' % slavename)

  manager = BuildSlaveManager(slavename, slave_dir, copies, os.pardir,
                              slave_cfg['master'],
                              slave_cfg.get('keepalive_conditions', []),
                              DEFAULT_POLL_INTERVAL)
  manager.start()

class FileLogger:
  """ Write stdout to a log file. """
  def __init__(self, log_file_name):
    # Open the log file.
    self._logfile = open(log_file_name, 'w')
    self._stdout = sys.stdout
    sys.stdout = self

  def __del__(self):
    self._logfile.close()
    sys.stdout = self._stdout

  def fileno(self):
    return self._stdout.fileno()

  def write(self, data):
    self._logfile.write(data)
    self._stdout.write(data)
    # Always flush the log file.
    self._logfile.flush()

  def flush(self):
    self._logfile.flush()
    self._stdout.flush()

def ParseArgs(argv):
  """ Parse and validate command-line arguments. """

  class CollectedArgs(object):
    pass

  usage = (
"""launch_slaves.py: Launch build slaves.
python launch_slaves.py

-h, --help          Show this message.
""")

  def Exit(error_msg=None):
    if error_msg:
      print error_msg
    print usage
    sys.exit(1)

  while argv:
    arg = argv.pop(0)
    if arg == '-h' or arg == '--help':
      Exit()
    else:
      Exit('Unknown argument: %s' % arg)
  return CollectedArgs()

def main():
  """ Launch local build slave instances """
  # Gather command-line arguments.
  ParseArgs(sys.argv[1:])

  # Sync the buildbot code.
  subprocess.check_call([GCLIENT, 'sync', '--force', '-j1'])

  # Obtain configuration information about this build slave host machine.
  slave_host = slave_hosts_cfg.GetSlaveHostConfig(socket.gethostname())
  slaves = slave_host['slaves']
  copies = slave_host['copies']
  print 'Attempting to launch build slaves:'
  for slavename, _ in slaves:
    print '  %s' % slavename

  # Launch the build slaves
  for slavename, _ in slaves:
    RunSlave(slavename, copies)

if '__main__' == __name__:
  # Pipe all output to a log file.
  logfile = 'launch_slaves.log'
  if os.path.isfile(logfile):
    num = 1
    new_filename = logfile + '.' + str(num)
    while os.path.isfile(new_filename):
      num += 1
      new_filename = logfile + '.' + str(num)
    os.rename(logfile, new_filename)
  logger = FileLogger(logfile)
  sys.exit(main())
<|endoftext|>


<commit_before>"""
Pages in Django can are served up with custom HTTP headers containing useful
information about those pages -- namely, the content type and object ID.

This module contains utility functions for retrieving and doing interesting
things with these special "X-Headers" (so called because the HTTP spec demands
that custom headers are prefxed with "X-").

Next time you're at slashdot.org, watch out for X-Fry and X-Bender. :)
"""

def populate_xheaders(request, response, model, object_id):
    """
    Adds the "X-Object-Type" and "X-Object-Id" headers to the given
    HttpResponse according to the given model and object_id -- but only if the
    given HttpRequest object has an IP address within the INTERNAL_IPS setting
    or if the request is from a logged in staff member.
    """
    from django.conf import settings
    if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or (request.user.is_authenticated() and request.user.is_staff):
        response['X-Object-Type'] = "%s.%s" % (model._meta.app_label, model._meta.object_name.lower())
        response['X-Object-Id'] = str(object_id)
<commit_msg>Fixed bug in [3786]. django/core/xheaders.py no longer assumes request.user exists<commit_after>"""
Pages in Django can are served up with custom HTTP headers containing useful
information about those pages -- namely, the content type and object ID.

This module contains utility functions for retrieving and doing interesting
things with these special "X-Headers" (so called because the HTTP spec demands
that custom headers are prefxed with "X-").

Next time you're at slashdot.org, watch out for X-Fry and X-Bender. :)
"""

def populate_xheaders(request, response, model, object_id):
    """
    Adds the "X-Object-Type" and "X-Object-Id" headers to the given
    HttpResponse according to the given model and object_id -- but only if the
    given HttpRequest object has an IP address within the INTERNAL_IPS setting
    or if the request is from a logged in staff member.
    """
    from django.conf import settings
    if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or (hasattr(request, 'user') and request.user.is_authenticated() and request.user.is_staff):
        response['X-Object-Type'] = "%s.%s" % (model._meta.app_label, model._meta.object_name.lower())
        response['X-Object-Id'] = str(object_id)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import numpy as np
import netCDF4 as nc

from base_grid import BaseGrid

class Mom1Grid(BaseGrid):

    def __init__(self, h_grid_def, v_grid_def=None, mask_file=None,
                    description='MOM tripolar'):
        """
        MOM 1 degree grid.
        """

        self.type = 'Arakawa B'

        with nc.Dataset(h_grid_def) as f:

            # Select points from double density horizontal grid.
            # t-cells.
            x_t = f.variables['x_T'][:]
            y_t = f.variables['y_T'][:]

            # u-cells.
            x_u = f.variables['x_C'][:]
            y_u = f.variables['y_C'][:]

            self.area_t = f.variables['area_T'][:]
            self.area_u = f.variables['area_C'][:]

            self.mask_t = 1 - f.variables['wet'][:]
            self.mask_u = 1 - f.variables['wet'][:]

            self.clon_t = f.variables['x_vert_T'][:]
            self.clat_t = f.variables['y_vert_T'][:]
            self.clon_u = f.variables['x_vert_C'][:]
            self.clat_u = f.variables['y_vert_C'][:]

        super(Mom1Grid, self).__init__(x_t, y_t, levels=[0], x_u=x_u, y_u=y_u,
                                        description=description)

    def set_mask(self):
        """
        Mask is read from file above.
        """
        pass

    def calc_areas(self):
        """
        Areas are read from file above.
        """
        pass

    def make_corners(self):
        """
        Corners are read from file above.
        """
<commit_msg>Put MOM1 mask the other way around.<commit_after>#!/usr/bin/env python

from __future__ import print_function

import numpy as np
import netCDF4 as nc

from base_grid import BaseGrid

class Mom1Grid(BaseGrid):

    def __init__(self, h_grid_def, v_grid_def=None, mask_file=None,
                    description='MOM tripolar'):
        """
        MOM 1 degree grid.
        """

        self.type = 'Arakawa B'

        with nc.Dataset(h_grid_def) as f:

            # Select points from double density horizontal grid.
            # t-cells.
            x_t = f.variables['x_T'][:]
            y_t = f.variables['y_T'][:]

            # u-cells.
            x_u = f.variables['x_C'][:]
            y_u = f.variables['y_C'][:]

            self.area_t = f.variables['area_T'][:]
            self.area_u = f.variables['area_C'][:]

            self.mask_t = f.variables['wet'][:]
            self.mask_u = f.variables['wet'][:]

            self.clon_t = f.variables['x_vert_T'][:]
            self.clat_t = f.variables['y_vert_T'][:]
            self.clon_u = f.variables['x_vert_C'][:]
            self.clat_u = f.variables['y_vert_C'][:]

        super(Mom1Grid, self).__init__(x_t, y_t, levels=[0], x_u=x_u, y_u=y_u,
                                        description=description)

    def set_mask(self):
        """
        Mask is read from file above.
        """
        pass

    def calc_areas(self):
        """
        Areas are read from file above.
        """
        pass

    def make_corners(self):
        """
        Corners are read from file above.
        """
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2012-2015 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import json

import six

from wirecloud.catalogue.models import CatalogueResource
from wirecloud.commons.utils.template.base import parse_contacts_info
from wirecloud.commons.utils.template.writers import rdf
from wirecloud.commons.utils.template.writers import xml
from wirecloud.platform.models import IWidget
from wirecloud.platform.wiring.utils import get_wiring_skeleton, parse_wiring_old_version
from wirecloud.platform.workspace.utils import VariableValueCacheManager

def get_iwidgets_description(included_iwidgets):
    description = "Wirecloud Mashup composed of: "

    description = ', '.join([iwidget.widget.resource.get_processed_info()['title'] for iwidget in included_iwidgets])

    return description

def get_workspace_description(workspace):
    included_iwidgets = IWidget.objects.filter(tab__workspace=workspace)

    return get_iwidgets_description(included_iwidgets)

def get_current_operator_pref_value(operator, preference):

    if preference['name'] in operator['preferences']:
        return "%s" % operator['preferences'][preference['name']]['value']
    else:
        return preference['default']

def process_iwidget(workspace, iwidget, wiring, parametrization, readOnlyWidgets):

    widget = iwidget.widget
    widget_description = widget.resource.get_template().get_resource_info()
    iwidget_id = str(iwidget.id)
    iwidget_params = {}
    if iwidget_id in parametrization:
        iwidget_params = parametrization[iwidget_id]

    cache_manager = VariableValueCacheManager(workspace, workspace.creator)

    # input and output endpoints
    for output_endpoint in widget_description['wiring']['outputs']:
        wiring['outputs'].append({
            'name': output_endpoint['name'],
            'type': output_endpoint['type'],
            'label': output_endpoint['label'],
            'description': output_endpoint['description'],
            'friendcode': output_endpoint['friendcode'],
        })

    for input_endpoint in widget_description['wiring']['inputs']:
        wiring['inputs'].append({
            'name': input_endpoint['name'],
            'type': input_endpoint['type'],
            'label': input_endpoint['label'],
            'description': input_endpoint['description'],
            'friendcode': input_endpoint['friendcode'],
            'actionlabel': input_endpoint['actionlabel'],
        })

    # preferences
    widget_preferences = widget_description['preferences']
    preferences = {}
    for pref in widget_preferences:
        status = 'normal'
        if pref['name'] in iwidget_params:
            iwidget_param_desc = iwidget_params[pref['name']]
            status = iwidget_param_desc.get('status', 'normal')
            source = iwidget_param_desc.get('source', 'current')
            if source == 'default':
                if status == 'normal':
                    # Do not issue a Preference element for this preference
                    continue
                value = None
            elif source == 'current':
                value = cache_manager.get_variable_value_from_varname(iwidget, pref['name'])
            elif source == 'custom':
                value = iwidget_param_desc['value']
            else:
                raise Exception('Invalid preference value source: %s' % source)

        else:
            value = cache_manager.get_variable_value_from_varname(iwidget, pref['name'])

        preferences[pref['name']] = {
            'readonly': status != 'normal',
            'hidden': status == 'hidden',
        }

        if value is not None:
            if pref['type'] == 'boolean':
                value = str(value).lower()
            elif pref['type'] == 'number':
                value = str(value)

            preferences[pref['name']]['value'] = value

    # iWidget properties
    widget_properties = widget_description['properties']
    properties = {}
    for prop in widget_properties:
        status = 'normal'
        if prop['name'] in iwidget_params:
            iwidget_param_desc = iwidget_params[prop['name']]
            status = iwidget_param_desc.get('status', 'normal')
            source = iwidget_param_desc.get('source', 'current')
            if source == 'default':
                if status == 'normal':
                    # Do not issue a Property element for this property
                    continue
                else:
                    value = None
            elif source == 'current':
                value = cache_manager.get_variable_value_from_varname(iwidget, prop['name'])
            elif source == 'custom':
                value = iwidget_param_desc['value']
            else:
                raise Exception('Invalid property value source: %s' % source)
        else:
            value = cache_manager.get_variable_value_from_varname(iwidget, prop['name'])

        properties[prop['name']] = {
            'readonly': status != 'normal',
            'value': value,
        }

    return {
        'id': iwidget_id,
        'vendor': iwidget.widget.resource.vendor,
        'name': iwidget.widget.resource.short_name,
        'version': iwidget.widget.resource.version,
        'title': iwidget.name,
        'readonly': readOnlyWidgets,
        'properties': properties,
        'preferences': preferences,
        'position': {
            'x': str(iwidget.positions['widget']['left']),
            'y': str(iwidget.positions['widget']['top']),
            'z': str(iwidget.positions['widget']['zIndex']),
        },
        'rendering': {
            'width': str(iwidget.positions['widget']['width']),
            'height': str(iwidget.positions['widget']['height']),
            'layout': str(iwidget.layout),
            'fulldragboard': str(iwidget.positions['widget']['fulldragboard']),
            'minimized': str(iwidget.positions['widget']['minimized']),
        },
    }

def build_json_template_from_workspace(options, workspace, user):
    options['type'] = 'mashup'
    options['params'] = []
    options['embedmacs'] = options.get('embedmacs', False) is True
    options['embedded'] = set()
    options['translations'] = {}
    options['translation_index_usage'] = {}

    description = options.get('description', '').strip()
    if description == '':
        options['description'] = get_workspace_description(workspace)

    if 'authors' not in options:
        options['authors'] = ({'name': six.text_type(user)},)
    elif isinstance(options['authors'], six.text_type):
        options['authors'] = parse_contacts_info(options['authors'])

    if 'contributors' not in options:
        options['contributors'] = ()
    elif isinstance(options['contributors'], six.text_type):
        options['contributors'] = parse_contacts_info(options['contributors'])

    options['requirements'] = []

    readOnlyWidgets = options.get('readOnlyWidgets', False)
    parametrization = options.get('parametrization')
    if not parametrization:
        parametrization = {}
    if 'iwidgets' not in parametrization:
        parametrization['iwidgets'] = {}
    if 'ioperators' not in parametrization:
        parametrization['ioperators'] = {}

    # Workspace preferences
    options['preferences'] = {}
    for preference in workspace.workspacepreference_set.all():
        if not preference.inherit:
            options['preferences'][preference.name] = preference.value

    # Tabs and their preferences
    options['tabs'] = []
    options['wiring'] = {
        'inputs': [],
        'outputs': [],
    }
    for tab in workspace.tab_set.order_by('position'):
        preferences = {}
        for preference in tab.tabpreference_set.all():
            if not preference.inherit:
                preferences[preference.name] = preference.value

        resources = []
        for iwidget in tab.iwidget_set.select_related('widget__resource', 'position').all():
            resource_info = process_iwidget(workspace, iwidget, options['wiring'], parametrization['iwidgets'], readOnlyWidgets)
            resources.append(resource_info)
            if options['embedmacs']:
                options['embedded'].add('/'.join((resource_info['vendor'], resource_info['name'], resource_info['version'])))

        options['tabs'].append({
            'name': tab.name,
            'resources': resources,
            'preferences': preferences,
        })

    # wiring conections and operators
    readOnlyConnectables = options.get('readOnlyConnectables', False)
    wiring_status = workspace.wiringStatus
    if len(wiring_status) == 0:
        wiring_status = get_wiring_skeleton()

    # Set the wiring status' version
    if wiring_status.get('version', '1.0') == '1.0':
        wiring_status = parse_wiring_old_version(wiring_status)

    options['wiring']['version'] = '2.0'
    options['wiring']['operators'] = {}
    for id_, operator in six.iteritems(wiring_status['operators']):
        operator_data = {
            'name': operator['name'],
            'preferences': {},
        }

        vendor, name, version = operator['name'].split('/')
        resource = CatalogueResource.objects.get(vendor=vendor, short_name=name, version=version)
        operator_info = json.loads(resource.json_description)
        operator_params = parametrization['ioperators'].get(id_, {})
        for pref_index, preference in enumerate(operator_info['preferences']):

            status = 'normal'
            if preference['name'] in operator_params:
                ioperator_param_desc = operator_params[preference['name']]
                status = ioperator_param_desc.get('status', 'normal')
                source = ioperator_param_desc.get('source', 'current')
                if source == 'default':
                    if status == 'normal':
                        # Do not issue a Preference element for this preference
                        continue
                    value = None
                elif source == 'current':
                    value = get_current_operator_pref_value(operator, preference)
                elif source == 'custom':
                    value = ioperator_param_desc['value']
                else:
                    raise Exception('Invalid preference value source: %s' % source)

            else:
                value = get_current_operator_pref_value(operator, preference)

            operator_data['preferences'][preference['name']] = {
                'readonly': status != 'normal',
                'hidden': status == 'hidden',
            }
            if value is not None:
                operator_data['preferences'][preference['name']]['value'] = value

        options['wiring']['operators'][id_] = operator_data
        if options['embedmacs']:
            options['embedded'].add(operator['name'])

    options['wiring']['connections'] = []
    for connection in wiring_status['connections']:
        options['wiring']['connections'].append({
            'source': connection['source'],
            'target': connection['target'],
            'readonly': readOnlyConnectables,
        })

    options['wiring']['visualdescription'] = wiring_status['visualdescription']

    embedded = options['embedded']
    options['embedded'] = []
    for resource in embedded:
        (vendor, name, version) = resource.split('/')
        options['embedded'].append({
            'vendor': vendor,
            'name': name,
            'version': version,
            'src': 'macs/%s_%s_%s.wgt' % (vendor, name, version)
        })
    del options['embedmacs']

    return options

def build_xml_template_from_workspace(options, workspace, user, raw=False):

    build_json_template_from_workspace(options, workspace, user)

    return xml.write_xml_description(options, raw=raw)

def build_rdf_template_from_workspace(options, workspace, user):

    build_json_template_from_workspace(options, workspace, user)

    return rdf.build_rdf_graph(options)
<commit_msg>Remove invalid field from select_related making Django 1.8 fail. See #85<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2012-2015 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import json

import six

from wirecloud.catalogue.models import CatalogueResource
from wirecloud.commons.utils.template.base import parse_contacts_info
from wirecloud.commons.utils.template.writers import rdf
from wirecloud.commons.utils.template.writers import xml
from wirecloud.platform.models import IWidget
from wirecloud.platform.wiring.utils import get_wiring_skeleton, parse_wiring_old_version
from wirecloud.platform.workspace.utils import VariableValueCacheManager

def get_iwidgets_description(included_iwidgets):
    description = "Wirecloud Mashup composed of: "

    description = ', '.join([iwidget.widget.resource.get_processed_info()['title'] for iwidget in included_iwidgets])

    return description

def get_workspace_description(workspace):
    included_iwidgets = IWidget.objects.filter(tab__workspace=workspace)

    return get_iwidgets_description(included_iwidgets)

def get_current_operator_pref_value(operator, preference):

    if preference['name'] in operator['preferences']:
        return "%s" % operator['preferences'][preference['name']]['value']
    else:
        return preference['default']

def process_iwidget(workspace, iwidget, wiring, parametrization, readOnlyWidgets):

    widget = iwidget.widget
    widget_description = widget.resource.get_template().get_resource_info()
    iwidget_id = str(iwidget.id)
    iwidget_params = {}
    if iwidget_id in parametrization:
        iwidget_params = parametrization[iwidget_id]

    cache_manager = VariableValueCacheManager(workspace, workspace.creator)

    # input and output endpoints
    for output_endpoint in widget_description['wiring']['outputs']:
        wiring['outputs'].append({
            'name': output_endpoint['name'],
            'type': output_endpoint['type'],
            'label': output_endpoint['label'],
            'description': output_endpoint['description'],
            'friendcode': output_endpoint['friendcode'],
        })

    for input_endpoint in widget_description['wiring']['inputs']:
        wiring['inputs'].append({
            'name': input_endpoint['name'],
            'type': input_endpoint['type'],
            'label': input_endpoint['label'],
            'description': input_endpoint['description'],
            'friendcode': input_endpoint['friendcode'],
            'actionlabel': input_endpoint['actionlabel'],
        })

    # preferences
    widget_preferences = widget_description['preferences']
    preferences = {}
    for pref in widget_preferences:
        status = 'normal'
        if pref['name'] in iwidget_params:
            iwidget_param_desc = iwidget_params[pref['name']]
            status = iwidget_param_desc.get('status', 'normal')
            source = iwidget_param_desc.get('source', 'current')
            if source == 'default':
                if status == 'normal':
                    # Do not issue a Preference element for this preference
                    continue
                value = None
            elif source == 'current':
                value = cache_manager.get_variable_value_from_varname(iwidget, pref['name'])
            elif source == 'custom':
                value = iwidget_param_desc['value']
            else:
                raise Exception('Invalid preference value source: %s' % source)

        else:
            value = cache_manager.get_variable_value_from_varname(iwidget, pref['name'])

        preferences[pref['name']] = {
            'readonly': status != 'normal',
            'hidden': status == 'hidden',
        }

        if value is not None:
            if pref['type'] == 'boolean':
                value = str(value).lower()
            elif pref['type'] == 'number':
                value = str(value)

            preferences[pref['name']]['value'] = value

    # iWidget properties
    widget_properties = widget_description['properties']
    properties = {}
    for prop in widget_properties:
        status = 'normal'
        if prop['name'] in iwidget_params:
            iwidget_param_desc = iwidget_params[prop['name']]
            status = iwidget_param_desc.get('status', 'normal')
            source = iwidget_param_desc.get('source', 'current')
            if source == 'default':
                if status == 'normal':
                    # Do not issue a Property element for this property
                    continue
                else:
                    value = None
            elif source == 'current':
                value = cache_manager.get_variable_value_from_varname(iwidget, prop['name'])
            elif source == 'custom':
                value = iwidget_param_desc['value']
            else:
                raise Exception('Invalid property value source: %s' % source)
        else:
            value = cache_manager.get_variable_value_from_varname(iwidget, prop['name'])

        properties[prop['name']] = {
            'readonly': status != 'normal',
            'value': value,
        }

    return {
        'id': iwidget_id,
        'vendor': iwidget.widget.resource.vendor,
        'name': iwidget.widget.resource.short_name,
        'version': iwidget.widget.resource.version,
        'title': iwidget.name,
        'readonly': readOnlyWidgets,
        'properties': properties,
        'preferences': preferences,
        'position': {
            'x': str(iwidget.positions['widget']['left']),
            'y': str(iwidget.positions['widget']['top']),
            'z': str(iwidget.positions['widget']['zIndex']),
        },
        'rendering': {
            'width': str(iwidget.positions['widget']['width']),
            'height': str(iwidget.positions['widget']['height']),
            'layout': str(iwidget.layout),
            'fulldragboard': str(iwidget.positions['widget']['fulldragboard']),
            'minimized': str(iwidget.positions['widget']['minimized']),
        },
    }

def build_json_template_from_workspace(options, workspace, user):
    options['type'] = 'mashup'
    options['params'] = []
    options['embedmacs'] = options.get('embedmacs', False) is True
    options['embedded'] = set()
    options['translations'] = {}
    options['translation_index_usage'] = {}

    description = options.get('description', '').strip()
    if description == '':
        options['description'] = get_workspace_description(workspace)

    if 'authors' not in options:
        options['authors'] = ({'name': six.text_type(user)},)
    elif isinstance(options['authors'], six.text_type):
        options['authors'] = parse_contacts_info(options['authors'])

    if 'contributors' not in options:
        options['contributors'] = ()
    elif isinstance(options['contributors'], six.text_type):
        options['contributors'] = parse_contacts_info(options['contributors'])

    options['requirements'] = []

    readOnlyWidgets = options.get('readOnlyWidgets', False)
    parametrization = options.get('parametrization')
    if not parametrization:
        parametrization = {}
    if 'iwidgets' not in parametrization:
        parametrization['iwidgets'] = {}
    if 'ioperators' not in parametrization:
        parametrization['ioperators'] = {}

    # Workspace preferences
    options['preferences'] = {}
    for preference in workspace.workspacepreference_set.all():
        if not preference.inherit:
            options['preferences'][preference.name] = preference.value

    # Tabs and their preferences
    options['tabs'] = []
    options['wiring'] = {
        'inputs': [],
        'outputs': [],
    }
    for tab in workspace.tab_set.order_by('position'):
        preferences = {}
        for preference in tab.tabpreference_set.all():
            if not preference.inherit:
                preferences[preference.name] = preference.value

        resources = []
        for iwidget in tab.iwidget_set.select_related('widget__resource').all():
            resource_info = process_iwidget(workspace, iwidget, options['wiring'], parametrization['iwidgets'], readOnlyWidgets)
            resources.append(resource_info)
            if options['embedmacs']:
                options['embedded'].add('/'.join((resource_info['vendor'], resource_info['name'], resource_info['version'])))

        options['tabs'].append({
            'name': tab.name,
            'resources': resources,
            'preferences': preferences,
        })

    # wiring conections and operators
    readOnlyConnectables = options.get('readOnlyConnectables', False)
    wiring_status = workspace.wiringStatus
    if len(wiring_status) == 0:
        wiring_status = get_wiring_skeleton()

    # Set the wiring status' version
    if wiring_status.get('version', '1.0') == '1.0':
        wiring_status = parse_wiring_old_version(wiring_status)

    options['wiring']['version'] = '2.0'
    options['wiring']['operators'] = {}
    for id_, operator in six.iteritems(wiring_status['operators']):
        operator_data = {
            'name': operator['name'],
            'preferences': {},
        }

        vendor, name, version = operator['name'].split('/')
        resource = CatalogueResource.objects.get(vendor=vendor, short_name=name, version=version)
        operator_info = json.loads(resource.json_description)
        operator_params = parametrization['ioperators'].get(id_, {})
        for pref_index, preference in enumerate(operator_info['preferences']):

            status = 'normal'
            if preference['name'] in operator_params:
                ioperator_param_desc = operator_params[preference['name']]
                status = ioperator_param_desc.get('status', 'normal')
                source = ioperator_param_desc.get('source', 'current')
                if source == 'default':
                    if status == 'normal':
                        # Do not issue a Preference element for this preference
                        continue
                    value = None
                elif source == 'current':
                    value = get_current_operator_pref_value(operator, preference)
                elif source == 'custom':
                    value = ioperator_param_desc['value']
                else:
                    raise Exception('Invalid preference value source: %s' % source)

            else:
                value = get_current_operator_pref_value(operator, preference)

            operator_data['preferences'][preference['name']] = {
                'readonly': status != 'normal',
                'hidden': status == 'hidden',
            }
            if value is not None:
                operator_data['preferences'][preference['name']]['value'] = value

        options['wiring']['operators'][id_] = operator_data
        if options['embedmacs']:
            options['embedded'].add(operator['name'])

    options['wiring']['connections'] = []
    for connection in wiring_status['connections']:
        options['wiring']['connections'].append({
            'source': connection['source'],
            'target': connection['target'],
            'readonly': readOnlyConnectables,
        })

    options['wiring']['visualdescription'] = wiring_status['visualdescription']

    embedded = options['embedded']
    options['embedded'] = []
    for resource in embedded:
        (vendor, name, version) = resource.split('/')
        options['embedded'].append({
            'vendor': vendor,
            'name': name,
            'version': version,
            'src': 'macs/%s_%s_%s.wgt' % (vendor, name, version)
        })
    del options['embedmacs']

    return options

def build_xml_template_from_workspace(options, workspace, user, raw=False):

    build_json_template_from_workspace(options, workspace, user)

    return xml.write_xml_description(options, raw=raw)

def build_rdf_template_from_workspace(options, workspace, user):

    build_json_template_from_workspace(options, workspace, user)

    return rdf.build_rdf_graph(options)
<|endoftext|>


<commit_before><commit_msg>fixed path<commit_after><|endoftext|>


<commit_before>from sublime import Region
from sublime_plugin import TextCommand

class MoveTextHorizCommand(TextCommand):
    def move_text_horiz(self, edit, direction):
        for region in self.view.sel():
            if region.empty():
                continue

            orig_region = region
            sel_region = Region(region.begin() + direction, region.end() + direction)

            if sel_region.a < 0 or sel_region.b > self.view.size():
                continue

            if direction < 0:
                dest_region = Region(region.begin() + direction, region.end())
                move_text = self.view.substr(region) + self.view.substr(Region(region.begin() + direction, region.begin()))
            else:
                dest_region = Region(region.begin(), region.end() + direction)
                move_text = self.view.substr(Region(region.end(), region.end() + direction)) + self.view.substr(region)

            # Remove selection from RegionSet
            self.view.sel().subtract(orig_region)
            # Replace the selection with transformed text
            self.view.replace(edit, dest_region, move_text)
            # Add the new selection
            self.view.sel().add(sel_region)

class MoveTextLeftCommand(MoveTextHorizCommand):
    def run(self, edit):
        e = self.view.begin_edit('move_text_horiz')
        self.move_text_horiz(edit, -1)
        self.view.end_edit(e)

class MoveTextRightCommand(MoveTextHorizCommand):
    def run(self, edit):
        e = self.view.begin_edit('move_text_horiz')
        self.move_text_horiz(edit, 1)
        self.view.end_edit(e)

class MoveTextVertCommand(TextCommand):
    def __init__(self, view):
        super(MoveTextVertCommand, self).__init__(view)
        view.move_text_vert_column = None

    def move_text_vert(self, region, edit, direction):
        if region.empty():
            if direction < 0:
                self.view.run_command('swap_line_up')
            else:
                self.view.run_command('swap_line_down')
            return

        orig_region = region
        move_text = self.view.substr(region)

        # calculate number of characters to the left
        row, col = self.view.rowcol(region.begin())

        # if the last command was a vertical move, use that column
        # the column is stored on the view - each command has its own instance,
        # and we don't want two buffers to modify the same object (e.g. MoveTextVertCommand)
        cmd, _, _ = self.view.command_history(0, True)
        if cmd != 'move_text_up' and cmd != 'move_text_down':
            self.view.move_text_vert_column = col
        else:
            col = self.view.move_text_vert_column

        dest_row = row + direction

        max_row = self.view.rowcol(self.view.size())[0]
        if dest_row < 0:
            dest_row = 0
        elif dest_row > max_row:
            dest_row = max_row

        self.view.sel().subtract(orig_region)
        self.view.replace(edit, region, '')

        # starting at the destination row at col 0, count off "col" characters
        # it's possible that there aren't enough characters in the destination row,
        # so stop if we end up on the wrong row, or past the buffer
        dest_point = self.view.text_point(dest_row, 0)
        if dest_point is None:
            dest_point = self.view.size()
        else:
            dest_line = self.view.line(dest_point)
            if dest_point + col > dest_line.b:
                dest_point = dest_line.b
            else:
                dest_point = dest_point + col

        insert_region = Region(dest_point, dest_point)
        sel_region = Region(dest_point, dest_point + len(move_text))

        self.view.replace(edit, insert_region, move_text)
        self.view.sel().add(sel_region)
        self.view.show(sel_region)

class MoveTextUpCommand(MoveTextVertCommand):
    def run(self, edit):
        e = self.view.begin_edit('move_text_vert')
        regions = [region for region in self.view.sel()]

        # sort by region.end() DESC
        def get_end(region):
            return region.end()
        regions.sort(key=get_end, reverse=True)

        for region in regions:
            self.move_text_vert(region, edit, -1)
        self.view.end_edit(e)

class MoveTextDownCommand(MoveTextVertCommand):
    def run(self, edit):
        e = self.view.begin_edit('move_text_vert')
        regions = [region for region in self.view.sel()]

        # sort by region.end() DESC
        def get_end(region):
            return region.end()
        regions.sort(key=get_end, reverse=True)

        for region in regions:
            self.move_text_vert(region, edit, 1)
        self.view.end_edit(e)
<commit_msg>more ST3 support<commit_after>from functools import cmp_to_key

from sublime import Region
from sublime_plugin import TextCommand

class MoveTextHorizCommand(TextCommand):
    def move_text_horiz(self, edit, direction):
        for region in self.view.sel():
            if region.empty():
                continue

            orig_region = region
            sel_region = Region(region.begin() + direction, region.end() + direction)

            if sel_region.a < 0 or sel_region.b > self.view.size():
                continue

            if direction < 0:
                dest_region = Region(region.begin() + direction, region.end())
                move_text = self.view.substr(region) + self.view.substr(Region(region.begin() + direction, region.begin()))
            else:
                dest_region = Region(region.begin(), region.end() + direction)
                move_text = self.view.substr(Region(region.end(), region.end() + direction)) + self.view.substr(region)

            # Remove selection from RegionSet
            self.view.sel().subtract(orig_region)
            # Replace the selection with transformed text
            self.view.replace(edit, dest_region, move_text)
            # Add the new selection
            self.view.sel().add(sel_region)

class MoveTextLeftCommand(MoveTextHorizCommand):
    def run(self, edit):
        self.move_text_horiz(edit, -1)

class MoveTextRightCommand(MoveTextHorizCommand):
    def run(self, edit):
        self.move_text_horiz(edit, 1)

class MoveTextVertCommand(TextCommand):
    def __init__(self, view):
        super(MoveTextVertCommand, self).__init__(view)
        view.move_text_vert_column = None

    def move_text_vert(self, region, edit, direction):
        if region.empty():
            if direction < 0:
                self.view.run_command('swap_line_up')
            else:
                self.view.run_command('swap_line_down')
            return

        orig_region = region
        move_text = self.view.substr(region)

        # calculate number of characters to the left
        row, col = self.view.rowcol(region.begin())

        # if the last command was a vertical move, use that column
        # the column is stored on the view - each command has its own instance,
        # and we don't want two buffers to modify the same object (e.g. MoveTextVertCommand)
        cmd, _, _ = self.view.command_history(0, True)
        if cmd != 'move_text_up' and cmd != 'move_text_down':
            self.view.move_text_vert_column = col
        else:
            col = self.view.move_text_vert_column

        dest_row = row + direction

        max_row = self.view.rowcol(self.view.size())[0]
        if dest_row < 0:
            dest_row = 0
        elif dest_row > max_row:
            dest_row = max_row

        self.view.sel().subtract(orig_region)
        self.view.replace(edit, region, '')

        # starting at the destination row at col 0, count off "col" characters
        # it's possible that there aren't enough characters in the destination row,
        # so stop if we end up on the wrong row, or past the buffer
        dest_point = self.view.text_point(dest_row, 0)
        if dest_point is None:
            dest_point = self.view.size()
        else:
            dest_line = self.view.line(dest_point)
            if dest_point + col > dest_line.b:
                dest_point = dest_line.b
            else:
                dest_point = dest_point + col

        insert_region = Region(dest_point, dest_point)
        sel_region = Region(dest_point, dest_point + len(move_text))

        self.view.replace(edit, insert_region, move_text)
        self.view.sel().add(sel_region)
        self.view.show(sel_region)

class MoveTextUpCommand(MoveTextVertCommand):
    def run(self, edit):
        regions = [region for region in self.view.sel()]

        # sort by region.end() DESC
        def get_end(region):
            return region.end()
        regions.sort(key=get_end, reverse=True)

        for region in regions:
            self.move_text_vert(region, edit, -1)

class MoveTextDownCommand(MoveTextVertCommand):
    def run(self, edit):
        regions = [region for region in self.view.sel()]

        # sort by region.end() DESC
        def get_end(region):
            return region.end()
        regions.sort(key=get_end, reverse=True)

        for region in regions:
            self.move_text_vert(region, edit, 1)
<|endoftext|>


<commit_before>"""
Scikit-learn wrapper interface of xgboost
"""

import xgboost as xgb
import numpy as np
import os
from deepchem.models import Model
from deepchem.models.sklearn_models import SklearnModel
from deepchem.utils.save import load_from_disk
from deepchem.utils.save import save_to_disk
from sklearn.cross_validation import train_test_split
from sklearn.grid_search import GridSearchCV
import tempfile

class XGBoostModel(SklearnModel):
  """
  Abstract base class for XGBoost model.
  """

  def __init__(self,
               model_instance=None,
               model_dir=None,
               verbose=False,
               **kwargs):
    """Abstract class for XGBoost models.
    Parameters:
    -----------
    model_instance: object
      Scikit-learn wrapper interface of xgboost
    model_dir: str
      Path to directory where model will be stored.
    """
    if model_dir is not None:
      if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    else:
      model_dir = tempfile.mkdtemp()
    self.model_dir = model_dir
    self.model_instance = model_instance
    self.model_class = model_instance.__class__

    self.verbose = verbose
    if 'early_stopping_rounds' in kwargs:
      self.early_stopping_rounds = kwargs['early_stopping_rounds']
    else:
      self.early_stopping_rounds = 50

  def fit(self, dataset, **kwargs):
    """
    Fits XGBoost model to data.
    """
    X = dataset.X
    y = np.squeeze(dataset.y)
    w = np.squeeze(dataset.w)
    seed = self.model_instance.seed
    if isinstance(self.model_instance, xgb.XGBClassifier):
      xgb_metric = "auc"
      sklearn_metric = "roc_auc"
      stratify = y
    elif isinstance(self.model_instance, xgb.XGBRegressor):
      xgb_metric = "mae"
      sklearn_metric = "neg_mean_absolute_error"
      stratify = None
    best_param = self._search_param(sklearn_metric, X, y)
    # update model with best param
    self.model_instance = self.model_class(**best_param)

    # Find optimal n_estimators based on original learning_rate
    # and early_stopping_rounds
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=seed, stratify=stratify)

    self.model_instance.fit(
        X_train,
        y_train,
        early_stopping_rounds=self.early_stopping_rounds,
        eval_metric=xgb_metric,
        eval_set=[(X_train, y_train), (X_test, y_test)],
        verbose=self.verbose)
    # Since test size is 20%, when retrain model to whole data, expect
    # n_estimator increased to 1/0.8 = 1.25 time.
    estimated_best_round = np.round(self.model_instance.best_ntree_limit * 1.25)
    self.model_instance.n_estimators = np.int64(estimated_best_round)
    self.model_instance.fit(
        X_train, y_train, eval_metric=xgb_metric, verbose=self.verbose)

  def _search_param(self, metric, X, y):
    '''
    Find best potential parameters set using few n_estimators
    '''
    # Make sure user specified params are in the grid.
    max_depth_grid = list(np.unique([self.model_instance.max_depth, 5, 7]))
    colsample_bytree_grid = list(
        np.unique([self.model_instance.colsample_bytree, 0.66, 0.9]))
    reg_lambda_grid = list(np.unique([self.model_instance.reg_lambda, 1, 5]))
    param_grid = {
        'max_depth': max_depth_grid,
        'learning_rate': [max(self.model_instance.learning_rate, 0.3)],
        'n_estimators': [min(self.model_instance.n_estimators, 60)],
        'gamma': [self.model_instance.gamma],
        'min_child_weight': [self.model_instance.min_child_weight],
        'max_delta_step': [self.model_instance.max_delta_step],
        'subsample': [self.model_instance.subsample],
        'colsample_bytree': colsample_bytree_grid,
        'colsample_bylevel': [self.model_instance.colsample_bylevel],
        'reg_alpha': [self.model_instance.reg_alpha],
        'reg_lambda': reg_lambda_grid,
        'scale_pos_weight': [self.model_instance.scale_pos_weight],
        'base_score': [self.model_instance.base_score],
        'seed': [self.model_instance.seed]
    }
    grid_search = GridSearchCV(
        self.model_instance, param_grid, cv=2, refit=False, scoring=metric)
    grid_search.fit(X, y)
    best_params = grid_search.best_params_
    # Change params back original params
    best_params['learning_rate'] = self.model_instance.learning_rate
    best_params['n_estimators'] = self.model_instance.n_estimators
    return best_params
<commit_msg>Update __init__.py<commit_after>"""
Scikit-learn wrapper interface of xgboost
"""

import xgboost as xgb
import numpy as np
import os
from deepchem.models import Model
from deepchem.models.sklearn_models import SklearnModel
from deepchem.utils.save import load_from_disk
from deepchem.utils.save import save_to_disk
from sklearn.cross_validation import train_test_split
from sklearn.grid_search import GridSearchCV
import tempfile

class XGBoostModel(SklearnModel):
  """
  Abstract base class for XGBoost model.
  """

  def __init__(self,
               model_instance=None,
               model_dir=None,
               verbose=False,
               **kwargs):
    """Abstract class for XGBoost models.
    Parameters:
    -----------
    model_instance: object
      Scikit-learn wrapper interface of xgboost
    model_dir: str
      Path to directory where model will be stored.
    """
    if model_dir is not None:
      if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    else:
      model_dir = tempfile.mkdtemp()
    self.model_dir = model_dir
    self.model_instance = model_instance
    self.model_class = model_instance.__class__

    self.verbose = verbose
    if 'early_stopping_rounds' in kwargs:
      self.early_stopping_rounds = kwargs['early_stopping_rounds']
    else:
      self.early_stopping_rounds = 50

  def fit(self, dataset, **kwargs):
    """
    Fits XGBoost model to data.
    """
    X = dataset.X
    y = np.squeeze(dataset.y)
    w = np.squeeze(dataset.w)
    seed = self.model_instance.seed
    if isinstance(self.model_instance, xgb.XGBClassifier):
      xgb_metric = "auc"
      sklearn_metric = "roc_auc"
      stratify = y
    elif isinstance(self.model_instance, xgb.XGBRegressor):
      xgb_metric = "mae"
      sklearn_metric = "neg_mean_absolute_error"
      stratify = None
    best_param = self._search_param(sklearn_metric, X, y)
    # update model with best param
    self.model_instance = self.model_class(**best_param)

    # Find optimal n_estimators based on original learning_rate
    # and early_stopping_rounds
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=seed, stratify=stratify)

    self.model_instance.fit(
        X_train,
        y_train,
        early_stopping_rounds=self.early_stopping_rounds,
        eval_metric=xgb_metric,
        eval_set=[(X_train, y_train), (X_test, y_test)],
        verbose=self.verbose)
    # Since test size is 20%, when retrain model to whole data, expect
    # n_estimator increased to 1/0.8 = 1.25 time.
    estimated_best_round = np.round(self.model_instance.best_ntree_limit * 1.25)
    self.model_instance.n_estimators = np.int64(estimated_best_round)
    self.model_instance.fit(X, y, eval_metric=xgb_metric, verbose=self.verbose)

  def _search_param(self, metric, X, y):
    '''
    Find best potential parameters set using few n_estimators
    '''
    # Make sure user specified params are in the grid.
    max_depth_grid = list(np.unique([self.model_instance.max_depth, 5, 7]))
    colsample_bytree_grid = list(
        np.unique([self.model_instance.colsample_bytree, 0.66, 0.9]))
    reg_lambda_grid = list(np.unique([self.model_instance.reg_lambda, 1, 5]))
    param_grid = {
        'max_depth': max_depth_grid,
        'learning_rate': [max(self.model_instance.learning_rate, 0.3)],
        'n_estimators': [min(self.model_instance.n_estimators, 60)],
        'gamma': [self.model_instance.gamma],
        'min_child_weight': [self.model_instance.min_child_weight],
        'max_delta_step': [self.model_instance.max_delta_step],
        'subsample': [self.model_instance.subsample],
        'colsample_bytree': colsample_bytree_grid,
        'colsample_bylevel': [self.model_instance.colsample_bylevel],
        'reg_alpha': [self.model_instance.reg_alpha],
        'reg_lambda': reg_lambda_grid,
        'scale_pos_weight': [self.model_instance.scale_pos_weight],
        'base_score': [self.model_instance.base_score],
        'seed': [self.model_instance.seed]
    }
    grid_search = GridSearchCV(
        self.model_instance, param_grid, cv=2, refit=False, scoring=metric)
    grid_search.fit(X, y)
    best_params = grid_search.best_params_
    # Change params back original params
    best_params['learning_rate'] = self.model_instance.learning_rate
    best_params['n_estimators'] = self.model_instance.n_estimators
    return best_params
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import csv
import sys

import numpy

from cerebro2.patcher import Patcher
from nupic.encoders.base import defaultDtype
from nupic.encoders.geospatial_coordinate import GeospatialCoordinateEncoder

def run(dataPath):
  encoder = GeospatialCoordinateEncoder(30,
                                        60,
                                        n=999,
                                        w=25,
                                        name='coordinate')
  Patcher().patchCoordinateEncoder(encoder, encoder.name)

  with open(dataPath) as csvfile:
    reader = csv.reader(csvfile)
    for row in reader:
      longitude = float(row[2])
      latitude = float(row[3])
      speed = float(row[5])
      encode(encoder, longitude, latitude, speed)

def encode(encoder, longitude, latitude, speed):
  output = numpy.zeros(encoder.getWidth(), dtype=defaultDtype)
  print ("===== " + str(latitude) + " / " + str(longitude) +
         " / " + str(speed) +  " =====")
  encoder.encodeIntoArray((longitude, latitude, speed), output)
  print output.nonzero()[0]
  print
  return output

if __name__ == "__main__":
  if len(sys.argv) < 2:
    print "Usage: {0} /path/to/data.csv".format(sys.argv[0])
    sys.exit(0)

  dataPath = sys.argv[1]
  run(dataPath)
<commit_msg>Better name<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import csv
import sys

import numpy

from cerebro2.patcher import Patcher
from nupic.encoders.base import defaultDtype
from nupic.encoders.geospatial_coordinate import GeospatialCoordinateEncoder

def run(dataPath):
  encoder = GeospatialCoordinateEncoder(30,
                                        60,
                                        n=999,
                                        w=25,
                                        name='geospatial_coordinate')
  Patcher().patchCoordinateEncoder(encoder, encoder.name)

  with open(dataPath) as csvfile:
    reader = csv.reader(csvfile)
    for row in reader:
      longitude = float(row[2])
      latitude = float(row[3])
      speed = float(row[5])
      encode(encoder, longitude, latitude, speed)

def encode(encoder, longitude, latitude, speed):
  output = numpy.zeros(encoder.getWidth(), dtype=defaultDtype)
  print ("===== " + str(latitude) + " / " + str(longitude) +
         " / " + str(speed) +  " =====")
  encoder.encodeIntoArray((longitude, latitude, speed), output)
  print output.nonzero()[0]
  print
  return output

if __name__ == "__main__":
  if len(sys.argv) < 2:
    print "Usage: {0} /path/to/data.csv".format(sys.argv[0])
    sys.exit(0)

  dataPath = sys.argv[1]
  run(dataPath)
<|endoftext|>


<commit_before># Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Table definitions."""
TABLES = dict(
    brand=dict(
        columns=dict(
            brand='text',
            company='text',
            facebook_url='text',
            is_former='integer',
            is_licensed='integer',
            is_prescription='integer',
            logo_url='text',
            tm='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['company', 'brand'],
    ),
    campaign=dict(
        columns=dict(
            author='text',
            author_url='text',
            campaign='text',
            campaign_id='text',
            contributors='text',
            copyright='text',
            date='text',
            donate_url='text',
            email='text',
            facebook_url='text',
            goal='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['campaign_id'],
    ),
    category=dict(
        columns=dict(
            brand='text',
            category='text',
            company='text',
            is_implied='integer',
        ),
        primary_key=['company', 'brand', 'category'],
    ),
    claim=dict(
        columns=dict(
            brand='text',
            campaign_id='text',
            claim='text',
            company='text',
            date='text',
            judgment='tinyint',
            scope='text',
            url='text',
        ),
        primary_key=['campaign_id', 'company', 'brand', 'scope', 'claim'],
    ),
    company=dict(
        columns=dict(
            company='text',
            company_full='text',
            email='text',
            facebook_url='text',
            feedback_url='text',
            hq_country='text',
            logo_url='text',
            parent_company='text',
            phone='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['company'],
    ),
    company_name=dict(
        columns=dict(
            company='text',
            company_name='text',
            is_alias='tinyint',
            is_full='tinyint',
        ),
        indexes=[
            ['company_name'],
        ],
        primary_key=['company', 'company_name'],
    ),
    rating=dict(
        columns=dict(
            brand='text',
            campaign_id='text',
            company='text',
            date='text',
            description='text',
            grade='text',
            judgment='tinyint',
            max_score='numeric',
            min_score='numeric',
            num_ranked='integer',
            rank='integer',
            scope='text',
            score='numeric',
            url='text',
        ),
        primary_key=['campaign_id', 'company', 'brand', 'scope'],
    ),
    scraper_brand_map=dict(
        columns=dict(
            brand='text',
            company='text',
            scraper_brand='text',
            scraper_company='text',
            scraper_id='text',
        ),
        indexes=[
            ['company', 'brand'],
        ],
        primary_key=['scraper_id', 'scraper_company', 'scraper_brand'],
    ),
    scraper=dict(
        columns=dict(
            last_scraped='text',
            scraper_id='text',
        ),
        primary_key=['scraper_id'],
    ),
    scraper_category_map=dict(
        columns=dict(
            category='text',
            scraper_category='text',
            scraper_id='text',
        ),
        indexes=[
            ['category'],
        ],
        primary_key=['scraper_id', 'scraper_category'],
    ),
    scraper_company_map=dict(
        columns=dict(
            company='text',
            scraper_company='text',
            scraper_id='text',
        ),
        indexes=[
            ['company'],
        ],
        primary_key=['scraper_id', 'scraper_company'],
    ),
    subcategory=dict(
        columns=dict(
            category='text',
            is_implied='tinyint',
            subcategory='text',
        ),
        primary_key=['category', 'subcategory'],
    ),
    url=dict(
        columns=dict(
            url='text',
            last_scraped='text',
            facebook_url='text',
            twitter_handle='text',
        ),
        indexes=[
            ['url'],
        ],
        output=False,
        primary_key=['scraper_id', 'url'],
    ),
)
<commit_msg>use tinyint for all is_* fields<commit_after># Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Table definitions."""
TABLES = dict(
    brand=dict(
        columns=dict(
            brand='text',
            company='text',
            facebook_url='text',
            is_former='tinyint',
            is_licensed='tinyint',
            is_prescription='tinyint',
            logo_url='text',
            tm='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['company', 'brand'],
    ),
    campaign=dict(
        columns=dict(
            author='text',
            author_url='text',
            campaign='text',
            campaign_id='text',
            contributors='text',
            copyright='text',
            date='text',
            donate_url='text',
            email='text',
            facebook_url='text',
            goal='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['campaign_id'],
    ),
    category=dict(
        columns=dict(
            brand='text',
            category='text',
            company='text',
            is_implied='tinyint',
        ),
        primary_key=['company', 'brand', 'category'],
    ),
    claim=dict(
        columns=dict(
            brand='text',
            campaign_id='text',
            claim='text',
            company='text',
            date='text',
            judgment='int',
            scope='text',
            url='text',
        ),
        primary_key=['campaign_id', 'company', 'brand', 'scope', 'claim'],
    ),
    company=dict(
        columns=dict(
            company='text',
            company_full='text',
            email='text',
            facebook_url='text',
            feedback_url='text',
            hq_country='text',
            logo_url='text',
            parent_company='text',
            phone='text',
            twitter_handle='text',
            url='text',
        ),
        primary_key=['company'],
    ),
    company_name=dict(
        columns=dict(
            company='text',
            company_name='text',
            is_alias='tinyint',
            is_full='tinyint',
        ),
        indexes=[
            ['company_name'],
        ],
        primary_key=['company', 'company_name'],
    ),
    rating=dict(
        columns=dict(
            brand='text',
            campaign_id='text',
            company='text',
            date='text',
            description='text',
            grade='text',
            judgment='tinyint',
            max_score='numeric',
            min_score='numeric',
            num_ranked='integer',
            rank='integer',
            scope='text',
            score='numeric',
            url='text',
        ),
        primary_key=['campaign_id', 'company', 'brand', 'scope'],
    ),
    scraper_brand_map=dict(
        columns=dict(
            brand='text',
            company='text',
            scraper_brand='text',
            scraper_company='text',
            scraper_id='text',
        ),
        indexes=[
            ['company', 'brand'],
        ],
        primary_key=['scraper_id', 'scraper_company', 'scraper_brand'],
    ),
    scraper=dict(
        columns=dict(
            last_scraped='text',
            scraper_id='text',
        ),
        primary_key=['scraper_id'],
    ),
    scraper_category_map=dict(
        columns=dict(
            category='text',
            scraper_category='text',
            scraper_id='text',
        ),
        indexes=[
            ['category'],
        ],
        primary_key=['scraper_id', 'scraper_category'],
    ),
    scraper_company_map=dict(
        columns=dict(
            company='text',
            scraper_company='text',
            scraper_id='text',
        ),
        indexes=[
            ['company'],
        ],
        primary_key=['scraper_id', 'scraper_company'],
    ),
    subcategory=dict(
        columns=dict(
            category='text',
            is_implied='tinyint',
            subcategory='text',
        ),
        primary_key=['category', 'subcategory'],
    ),
    url=dict(
        columns=dict(
            url='text',
            last_scraped='text',
            facebook_url='text',
            twitter_handle='text',
        ),
        indexes=[
            ['url'],
        ],
        output=False,
        primary_key=['scraper_id', 'url'],
    ),
)
<|endoftext|>