<commit_before>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# (c) 2013 Dmitry Kondrashkin <kondra2lp@gmail.com>
#

import numpy as np

from pystruct.learners.ssvm import BaseSSVM
from pystruct.learners.n_slack_ssvm import NSlackSSVM
from pystruct.utils import find_constraint

class LatentSSVM(BaseSSVM):
    """Stuctured SVM solver for latent-variable models.

    This is a hard-EM-style algorithm that alternates between
    latent variable completion for the ground truth and
    learning the parameters with the latent variables held fixed
    using the ``base_ssvm`` solver.

    If the base_ssvm is an n-slack SSVM, the current constraints
    will be adjusted after recomputing the latent variables H.
    If the base_ssvm is a 1-slack SSVM, the inference cache will
    be reused. Both methods drastically speed up learning.

    Parameters
    ----------
    base_ssvm : object
        SSVM solver instance for solving the completed problem.

    latent_iter : int (default=5)
        Number of iterations in the completion / refit loop.

    logger : object
        Logger instance.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.
    """

    def __init__(self, base_ssvm, latent_iter=5, verbose=0, logger=None):
        self.base_ssvm = base_ssvm
        self.latent_iter = latent_iter
        self.logger = logger
        self.verbose = verbose

    def fit(self, X, Y, H_def, initialize=True):
        """Learn parameters using the concave-convex procedure.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        H_def: iterable
            Labels for full-labeles samples. (known hidden variables)
            Used for heterogenous training.
        """

        self.model.initialize(X, Y)
        w = np.zeros(self.model.size_psi)
        constraints = None
        ws = []

        X1 = []
        H1 = []
        H = H_def

        for i, h in enumerate(H_def):
            if h is not None:
                X1.append(X[i])
                H1.append(h)

        # all data is fully labeled, quit
        if len(X1) == len(X):
            self.base_ssvm.fit(X1, H1)
            w = self.base_ssvm.w
            return

        if initialize & len(X1) > 0:
            self.base_ssvm.fit(X1, H1)
            w = self.base_ssvm.w

        for iteration in xrange(self.latent_iter):
            if self.verbose:
                print("LATENT SVM ITERATION %d" % iteration)
            # find latent variables for ground truth:
            H_new = []
            for x, y, h in zip(X, Y, H_def):
                if h is None:
                    H_new.append(self.model.latent(x, y, w))
                else:
                    H_new.append(h)

            changes = [np.any(h_new != h) for h_new, h in zip(H_new, H)]
            if not np.any(changes):
                if self.verbose:
                    print("no changes in latent variables of ground truth."
                          " stopping.")
                break
            if self.verbose:
                print("changes in H: %d" % np.sum(changes))

            # update constraints:
            if isinstance(self.base_ssvm, NSlackSSVM):
                constraints = [[] for i in xrange(len(X))]
                for sample, h, i in zip(self.base_ssvm.constraints_, H_new,
                                        np.arange(len(X))):
                    for constraint in sample:
                        const = find_constraint(self.model, X[i], h, w,
                                                constraint[0])
                        y_hat, dpsi, _, loss = const
                        constraints[i].append([y_hat, dpsi, loss])
            H = H_new

            if iteration > 0:
                self.base_ssvm.fit(X, H, constraints=constraints,
                                   warm_start="soft", initialize=False)
            else:
                self.base_ssvm.fit(X, H, constraints=constraints,
                                   initialize=False)
            w = self.base_ssvm.w
            ws.append(w)
            if self.logger is not None:
                self.logger(self, iteration)

    def predict(self, X):
        prediction = self.base_ssvm.predict(X)
        return [self.model.label_from_latent(h) for h in prediction]

    def predict_latent(self, X):
        return self.base_ssvm.predict(X)

    def score(self, X, Y):
        """Compute score as 1 - loss over whole data set.

        Returns the average accuracy (in terms of model.loss)
        over X and Y.

        Parameters
        ----------
        X : iterable
            Evaluation data.

        Y : iterable
            True labels.

        Returns
        -------
        score : float
            Average of 1 - loss over training examples.
        """
        losses = [self.model.base_loss(y, y_pred)
                  for y, y_pred in zip(Y, self.predict(X))]
        max_losses = [self.model.max_loss(y) for y in Y]
        return 1. - np.sum(losses) / float(np.sum(max_losses))

    @property
    def model(self):
        return self.base_ssvm.model

    @model.setter
    def model(self, model_):
        self.base_ssvm.model = model_

    @property
    def w(self):
        return self.base_ssvm.w

    @w.setter
    def w(self, w_):
        self.base_ssvm.w = w_

    @property
    def C(self):
        return self.base_ssvm.C

    @C.setter
    def C(self, C_):
        self.base_ssvm.w = C_

    @property
    def n_jobs(self):
        return self.base_ssvm.n_jobs

    @n_jobs.setter
    def n_jobs(self, n_jobs_):
        self.base_ssvm.n_jobs = n_jobs_
<commit_msg>add support for hCRF<commit_after>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# (c) 2013 Dmitry Kondrashkin <kondra2lp@gmail.com>
#

import numpy as np

from pystruct.learners.ssvm import BaseSSVM
from pystruct.learners.n_slack_ssvm import NSlackSSVM
from pystruct.utils import find_constraint

class LatentSSVM(BaseSSVM):
    """Stuctured SVM solver for latent-variable models.

    This is a hard-EM-style algorithm that alternates between
    latent variable completion for the ground truth and
    learning the parameters with the latent variables held fixed
    using the ``base_ssvm`` solver.

    If the base_ssvm is an n-slack SSVM, the current constraints
    will be adjusted after recomputing the latent variables H.
    If the base_ssvm is a 1-slack SSVM, the inference cache will
    be reused. Both methods drastically speed up learning.

    Parameters
    ----------
    base_ssvm : object
        SSVM solver instance for solving the completed problem.

    latent_iter : int (default=5)
        Number of iterations in the completion / refit loop.

    logger : object
        Logger instance.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.
    """

    def __init__(self, base_ssvm, latent_iter=5, verbose=0, logger=None):
        self.base_ssvm = base_ssvm
        self.latent_iter = latent_iter
        self.logger = logger
        self.verbose = verbose

    def fit(self, X, Y, H_def, initialize=True, pass_labels=False):
        """Learn parameters using the concave-convex procedure.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        H_def: iterable
            Labels for full-labeles samples. (known hidden variables)
            Used for heterogenous training.

        initialize: boolean
            Initialize w by first running SSVM.

        pass_labels: boolean
            Pass hidden and original labels to SSVM solver as a tuple.
            Only use this if you know what you are doing!
        """

        self.model.initialize(X, Y)
        w = np.zeros(self.model.size_psi)
        constraints = None
        ws = []

        X1 = []
        H1 = []
        H = H_def

        for i, h in enumerate(H_def):
            if h is not None:
                X1.append(X[i])
                H1.append(h)
                Y[i] = None

        # all data is fully labeled, quit
        if len(X1) == len(X):
            self.base_ssvm.fit(X1, H1)
            w = self.base_ssvm.w
            return

        if initialize and len(X1) > 0:
            self.base_ssvm.fit(X1, H1)
            w = self.base_ssvm.w

        for iteration in xrange(self.latent_iter):
            if self.verbose:
                print("LATENT SVM ITERATION %d" % iteration)
            # find latent variables for ground truth:
            H_new = []
            for x, y, h in zip(X, Y, H_def):
                if h is None:
                    H_new.append(self.model.latent(x, y, w))
                else:
                    H_new.append(h)

            changes = [np.any(h_new != h) for h_new, h in zip(H_new, H)]
            if not np.any(changes):
                if self.verbose:
                    print("no changes in latent variables of ground truth."
                          " stopping.")
                break
            if self.verbose:
                print("changes in H: %d" % np.sum(changes))

            # update constraints:
            if isinstance(self.base_ssvm, NSlackSSVM):
                constraints = [[] for i in xrange(len(X))]
                for sample, h, i in zip(self.base_ssvm.constraints_, H_new,
                                        np.arange(len(X))):
                    for constraint in sample:
                        const = find_constraint(self.model, X[i], h, w,
                                                constraint[0])
                        y_hat, dpsi, _, loss = const
                        constraints[i].append([y_hat, dpsi, loss])
            H = H_new

            if pass_labels:
                T = zip(H, Y)
            else:
                T = H

            if iteration > 0:
                self.base_ssvm.fit(X, T, constraints=constraints,
                                   warm_start="soft", initialize=False)
            else:
                self.base_ssvm.fit(X, T, constraints=constraints,
                                   initialize=False)
            w = self.base_ssvm.w
            ws.append(w)
            if self.logger is not None:
                self.logger(self, iteration)

    def predict(self, X):
        prediction = self.base_ssvm.predict(X)
        return [self.model.label_from_latent(h) for h in prediction]

    def predict_latent(self, X):
        return self.base_ssvm.predict(X)

    def score(self, X, Y):
        """Compute score as 1 - loss over whole data set.

        Returns the average accuracy (in terms of model.loss)
        over X and Y.

        Parameters
        ----------
        X : iterable
            Evaluation data.

        Y : iterable
            True labels.

        Returns
        -------
        score : float
            Average of 1 - loss over training examples.
        """
        losses = [self.model.base_loss(y, y_pred)
                  for y, y_pred in zip(Y, self.predict(X))]
        max_losses = [self.model.max_loss(y) for y in Y]
        return 1. - np.sum(losses) / float(np.sum(max_losses))

    @property
    def model(self):
        return self.base_ssvm.model

    @model.setter
    def model(self, model_):
        self.base_ssvm.model = model_

    @property
    def w(self):
        return self.base_ssvm.w

    @w.setter
    def w(self, w_):
        self.base_ssvm.w = w_

    @property
    def C(self):
        return self.base_ssvm.C

    @C.setter
    def C(self, C_):
        self.base_ssvm.w = C_

    @property
    def n_jobs(self):
        return self.base_ssvm.n_jobs

    @n_jobs.setter
    def n_jobs(self, n_jobs_):
        self.base_ssvm.n_jobs = n_jobs_
<|endoftext|>


<commit_before>import urllib2
from StringIO import StringIO
from PIL import Image as pil_image

try:
    from collections import namedtuple
    Size = namedtuple('Size', 'width height')
except ImportError:
    class Size(object):
        def __init__(self, width, height):
            self.width = width
            self.height = height

        def __getitem__(self, index):
            if index == 0:
                return self.width
            if index == 1:
                return self.height
            raise IndexError()

        def __len__(self):
            return 2

        def __eq__(self, other):
            if isinstance(other, Size):
                return (
                    self.width == other.width and
                    self.height == other.height
                )
            return (
                len(other) == 2 and
                self[0] == other[0] and
                self[1] == other[1]
            )

class Image(object):
    def __init__(self, source, copy=True):
        if isinstance(source, str) or isinstance(source, unicode):
            if source.startswith('https://') or source.startswith('http://'):
                self._load_from_url(source)
            else:
                self._load_from_file_path(source)
        elif isinstance(source, pil_image.Image):
            self._load_from_pil_image(source, copy)
        elif isinstance(source, Image):
            self._load_from_image(source, copy)
        else:
            self._load_from_file_object(source)

        format = self._pil_image.format
        self.ext = format.lower() if format else None

    @property
    def size(self):
        return Size(*self._pil_image.size)

    @property
    def width(self):
        return self.size.width

    @property
    def height(self):
        return self.size.height

    def _load_from_image(self, source, copy):
        self._load_from_pil_image(source._pil_image, copy)

    def _load_from_pil_image(self, source, copy):
        if copy:
            self._pil_image = source.copy()
        else:
            self._pil_image = source

    def _load_from_file_path(self, source):
        self._pil_image = pil_image.open(source)

    def _load_from_url(self, source):
        self._load_from_file_object(StringIO(urllib2.urlopen(source).read()))

    def _load_from_file_object(self, source):
        self._pil_image = pil_image.open(source)

    def __getattr__(self, attr):
        return getattr(self._pil_image, attr)
<commit_msg>Add __str__ and __repr__ to Size. Closes #1.<commit_after>import urllib2
from StringIO import StringIO
from PIL import Image as pil_image

try:
    from collections import namedtuple
    Size = namedtuple('Size', 'width height')
except ImportError:
    class Size(object):
        def __init__(self, width, height):
            self.width = width
            self.height = height

        def __getitem__(self, index):
            if index == 0:
                return self.width
            if index == 1:
                return self.height
            raise IndexError()

        def __len__(self):
            return 2

        def __eq__(self, other):
            if isinstance(other, Size):
                return (
                    self.width == other.width and
                    self.height == other.height
                )
            return (
                len(other) == 2 and
                self[0] == other[0] and
                self[1] == other[1]
            )

        def __str__(self):
            return 'Size(width=%s, height=%s)' % (self[0], self[1])

        def __repr__(self):
            return str(self)

class Image(object):
    def __init__(self, source, copy=True):
        if isinstance(source, str) or isinstance(source, unicode):
            if source.startswith('https://') or source.startswith('http://'):
                self._load_from_url(source)
            else:
                self._load_from_file_path(source)
        elif isinstance(source, pil_image.Image):
            self._load_from_pil_image(source, copy)
        elif isinstance(source, Image):
            self._load_from_image(source, copy)
        else:
            self._load_from_file_object(source)

        format = self._pil_image.format
        self.ext = format.lower() if format else None

    @property
    def size(self):
        return Size(*self._pil_image.size)

    @property
    def width(self):
        return self.size.width

    @property
    def height(self):
        return self.size.height

    def _load_from_image(self, source, copy):
        self._load_from_pil_image(source._pil_image, copy)

    def _load_from_pil_image(self, source, copy):
        if copy:
            self._pil_image = source.copy()
        else:
            self._pil_image = source

    def _load_from_file_path(self, source):
        self._pil_image = pil_image.open(source)

    def _load_from_url(self, source):
        self._load_from_file_object(StringIO(urllib2.urlopen(source).read()))

    def _load_from_file_object(self, source):
        self._pil_image = pil_image.open(source)

    def __getattr__(self, attr):
        return getattr(self._pil_image, attr)
<|endoftext|>


<commit_before># Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Documentation on PRESUBMIT.py can be found at:
# http://www.chromium.org/developers/how-tos/depottools/presubmit-scripts

import os.path
import sys

# List of directories to not apply presubmit project checks, relative
# to the NaCl top directory
EXCLUDE_PROJECT_CHECKS_DIRS = [
    # The following contain test data (including automatically generated),
    # and do not follow our conventions.
    'src/trusted/validator_ragel/testdata/32',
    'src/trusted/validator_ragel/testdata/64',
    'src/trusted/validator_x86/testdata/32',
    'src/trusted/validator_x86/testdata/64',
    'src/trusted/validator/x86/decoder/generator/testdata/32',
    'src/trusted/validator/x86/decoder/generator/testdata/64',
    # The following directories contains automatically generated source,
    # which may not follow our conventions.
    'src/trusted/validator_x86/gen',
    'src/trusted/validator/x86/decoder/gen',
    'src/trusted/validator/x86/decoder/generator/gen',
    'src/trusted/validator/x86/ncval_seg_sfi/gen',
    'src/trusted/validator_arm/gen',
    'src/trusted/validator_ragel/gen',
    ]

def NaClTopDir():
  # git-cl and gcl run PRESUBMIT.py with the current directory set to
  # the native_client directory (which may have a different basename
  # in a standalone checkout).
  return os.getcwd()

def _CommonChecks(input_api, output_api):
  """Checks for both upload and commit."""
  results = []
  results.extend(input_api.canned_checks.PanProjectChecks(
      input_api, output_api, project_name='Native Client',
      excluded_paths=tuple(EXCLUDE_PROJECT_CHECKS_DIRS)))
  return results

def IsFileInDirectories(f, dirs):
  """ Returns true if f is in list of directories"""
  for d in dirs:
    if d is os.path.commonprefix([f , d]):
      return True
  return False

def CheckChangeOnUpload(input_api, output_api):
  """Verifies all changes in all files.
  Args:
    input_api: the limited set of input modules allowed in presubmit.
    output_api: the limited set of output modules allowed in presubmit.
  """
  report = []
  report.extend(_CommonChecks(input_api, output_api))

  # The commit queue assumes PRESUBMIT.py is standalone.
  # TODO(bradnelson): Migrate code_hygiene to a common location so that
  # it can be used by the commit queue.
  old_sys_path = list(sys.path)
  try:
    sys.path.append(os.path.join(NaClTopDir(), 'tools'))
    sys.path.append(os.path.join(NaClTopDir(), 'build'))
    import code_hygiene
  finally:
    sys.path = old_sys_path
    del old_sys_path

  affected_files = input_api.AffectedFiles(include_deletes=False)
  exclude_dirs = [ NaClTopDir() + '/' + x + '/'
                   for x in EXCLUDE_PROJECT_CHECKS_DIRS ]
  for filename in affected_files:
    filename = filename.AbsoluteLocalPath()
    if not IsFileInDirectories(filename, exclude_dirs):
      errors, warnings = code_hygiene.CheckFile(filename, False)
      for e in errors:
        report.append(output_api.PresubmitError(e, items=errors[e]))
      for w in warnings:
        report.append(output_api.PresubmitPromptWarning(w, items=warnings[w]))

  return report

def CheckChangeOnCommit(input_api, output_api):
  """Verifies all changes in all files and verifies that the
  tree is open and can accept a commit.
  Args:
    input_api: the limited set of input modules allowed in presubmit.
    output_api: the limited set of output modules allowed in presubmit.
  """
  report = []
  report.extend(CheckChangeOnUpload(input_api, output_api))
  report.extend(input_api.canned_checks.CheckTreeIsOpen(
      input_api, output_api,
      json_url='http://nativeclient-status.appspot.com/current?format=json'))
  return report

def GetPreferredTrySlaves(_, change):

  # This is a heuristic to (conservatively) detect if the change only
  # affects the pnacl toolchain and not the main builders, and run the
  # appropriate trybots.
  is_pnacl_toolchain_only = True
  for f in change.AffectedFiles(include_dirs=True):
    if not f.LocalPath().startswith('pnacl'):
      is_pnacl_toolchain_only = False
  if is_pnacl_toolchain_only:
    return [
        'nacl-toolchain-linux-pnacl-x86_64',
        'nacl-toolchain-linux-pnacl-x86_32',
        'nacl-toolchain-mac-pnacl-x86_32',
        'nacl-toolchain-win7-pnacl-x86_64',
    ]

  # If the change only affects the new toolchain_build scheme,
  # then run only the trybots doing that build.
  if all(IsFileInDirectories(file.AbsoluteLocalPath(),
                             [os.path.join(NaClTopDir(), 'toolchain_build')])
         for file in change.AffectedFiles(include_dirs=True)):
    return [
        'nacl-toolchain-precise64-newlib-arm',
        'nacl-toolchain-mac-newlib-arm',
        'nacl-toolchain-win7-newlib-arm',
        ]

  return [
      'nacl-precise32_newlib_dbg',
      'nacl-precise32_newlib_opt',
      'nacl-precise32_glibc_opt',
      'nacl-precise64_newlib_dbg',
      'nacl-precise64_newlib_opt',
      'nacl-precise64_glibc_opt',
      'nacl-mac10.6_newlib_opt',
      'nacl-mac10.6_glibc_opt',
      'nacl-mac10.6_64_newlib_dbg',
      'nacl-mac10.6_64_glibc_opt',
      'nacl-mac10.7_newlib_opt',
      'nacl-mac10.7_glibc_opt',
      'nacl-mac10.7_64_newlib_dbg',
      'nacl-mac10.7_64_glibc_opt',
      'nacl-mac10.8_32_newlib_dbg',
      'nacl-mac10.8_32_glibc_opt',
      'nacl-mac10.8_64_newlib_dbg',
      'nacl-mac10.8_64_glibc_opt',
      'nacl-win32_newlib_opt',
      'nacl-win32_glibc_opt',
      'nacl-win64_newlib_dbg',
      'nacl-win64_newlib_opt',
      'nacl-win64_glibc_opt',
      'nacl-win8-64_newlib_dbg',
      'nacl-win8-64_newlib_opt',
      'nacl-arm_opt_panda',
      # arm-nacl-gcc bots
      'nacl-win7_64_arm_newlib_opt',
      'nacl-mac10.7_arm_newlib_opt',
      'nacl-precise64_arm_newlib_opt',
      # Clang bots
      'nacl-precise_64-newlib-dbg-clang',
      'nacl-mac10.6-newlib-dbg-clang',
      # pnacl scons bots
      'nacl-precise_64-newlib-arm_qemu-pnacl',
      'nacl-precise_64-newlib-x86_32-pnacl',
      'nacl-precise_64-newlib-x86_64-pnacl',
      # pnacl spec2k bots
      'nacl-arm_perf_panda',
      'nacl-precise_64-newlib-x86_32-pnacl-spec',
      'nacl-precise_64-newlib-x86_64-pnacl-spec',
  ]
<commit_msg>Clean up trybot selection<commit_after># Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Documentation on PRESUBMIT.py can be found at:
# http://www.chromium.org/developers/how-tos/depottools/presubmit-scripts

import os.path
import sys

# List of directories to not apply presubmit project checks, relative
# to the NaCl top directory
EXCLUDE_PROJECT_CHECKS_DIRS = [
    # The following contain test data (including automatically generated),
    # and do not follow our conventions.
    'src/trusted/validator_ragel/testdata/32',
    'src/trusted/validator_ragel/testdata/64',
    'src/trusted/validator_x86/testdata/32',
    'src/trusted/validator_x86/testdata/64',
    'src/trusted/validator/x86/decoder/generator/testdata/32',
    'src/trusted/validator/x86/decoder/generator/testdata/64',
    # The following directories contains automatically generated source,
    # which may not follow our conventions.
    'src/trusted/validator_x86/gen',
    'src/trusted/validator/x86/decoder/gen',
    'src/trusted/validator/x86/decoder/generator/gen',
    'src/trusted/validator/x86/ncval_seg_sfi/gen',
    'src/trusted/validator_arm/gen',
    'src/trusted/validator_ragel/gen',
    ]

def NaClTopDir():
  # git-cl and gcl run PRESUBMIT.py with the current directory set to
  # the native_client directory (which may have a different basename
  # in a standalone checkout).
  return os.getcwd()

def _CommonChecks(input_api, output_api):
  """Checks for both upload and commit."""
  results = []
  results.extend(input_api.canned_checks.PanProjectChecks(
      input_api, output_api, project_name='Native Client',
      excluded_paths=tuple(EXCLUDE_PROJECT_CHECKS_DIRS)))
  return results

def IsFileInDirectories(f, dirs):
  """ Returns true if f is in list of directories"""
  for d in dirs:
    if d is os.path.commonprefix([f , d]):
      return True
  return False

def CheckChangeOnUpload(input_api, output_api):
  """Verifies all changes in all files.
  Args:
    input_api: the limited set of input modules allowed in presubmit.
    output_api: the limited set of output modules allowed in presubmit.
  """
  report = []
  report.extend(_CommonChecks(input_api, output_api))

  # The commit queue assumes PRESUBMIT.py is standalone.
  # TODO(bradnelson): Migrate code_hygiene to a common location so that
  # it can be used by the commit queue.
  old_sys_path = list(sys.path)
  try:
    sys.path.append(os.path.join(NaClTopDir(), 'tools'))
    sys.path.append(os.path.join(NaClTopDir(), 'build'))
    import code_hygiene
  finally:
    sys.path = old_sys_path
    del old_sys_path

  affected_files = input_api.AffectedFiles(include_deletes=False)
  exclude_dirs = [ NaClTopDir() + '/' + x + '/'
                   for x in EXCLUDE_PROJECT_CHECKS_DIRS ]
  for filename in affected_files:
    filename = filename.AbsoluteLocalPath()
    if not IsFileInDirectories(filename, exclude_dirs):
      errors, warnings = code_hygiene.CheckFile(filename, False)
      for e in errors:
        report.append(output_api.PresubmitError(e, items=errors[e]))
      for w in warnings:
        report.append(output_api.PresubmitPromptWarning(w, items=warnings[w]))

  return report

def CheckChangeOnCommit(input_api, output_api):
  """Verifies all changes in all files and verifies that the
  tree is open and can accept a commit.
  Args:
    input_api: the limited set of input modules allowed in presubmit.
    output_api: the limited set of output modules allowed in presubmit.
  """
  report = []
  report.extend(CheckChangeOnUpload(input_api, output_api))
  report.extend(input_api.canned_checks.CheckTreeIsOpen(
      input_api, output_api,
      json_url='http://nativeclient-status.appspot.com/current?format=json'))
  return report

DEFAULT_TRYBOTS = [
    'nacl-precise32_newlib_dbg',
    'nacl-precise32_newlib_opt',
    'nacl-precise32_glibc_opt',
    'nacl-precise64_newlib_dbg',
    'nacl-precise64_newlib_opt',
    'nacl-precise64_glibc_opt',
    'nacl-mac10.6_newlib_opt',
    'nacl-mac10.6_glibc_opt',
    'nacl-mac10.6_64_newlib_dbg',
    'nacl-mac10.6_64_glibc_opt',
    'nacl-mac10.7_newlib_opt',
    'nacl-mac10.7_glibc_opt',
    'nacl-mac10.7_64_newlib_dbg',
    'nacl-mac10.7_64_glibc_opt',
    'nacl-mac10.8_32_newlib_dbg',
    'nacl-mac10.8_32_glibc_opt',
    'nacl-mac10.8_64_newlib_dbg',
    'nacl-mac10.8_64_glibc_opt',
    'nacl-win32_newlib_opt',
    'nacl-win32_glibc_opt',
    'nacl-win64_newlib_dbg',
    'nacl-win64_newlib_opt',
    'nacl-win64_glibc_opt',
    'nacl-win8-64_newlib_dbg',
    'nacl-win8-64_newlib_opt',
    'nacl-arm_opt_panda',
    # arm-nacl-gcc bots
    'nacl-win7_64_arm_newlib_opt',
    'nacl-mac10.7_arm_newlib_opt',
    'nacl-precise64_arm_newlib_opt',
    # Clang bots
    'nacl-precise_64-newlib-dbg-clang',
    'nacl-mac10.6-newlib-dbg-clang',
    # pnacl scons bots
    'nacl-precise_64-newlib-arm_qemu-pnacl',
    'nacl-precise_64-newlib-x86_32-pnacl',
    'nacl-precise_64-newlib-x86_64-pnacl',
    # pnacl spec2k bots
    'nacl-arm_perf_panda',
    'nacl-precise_64-newlib-x86_32-pnacl-spec',
    'nacl-precise_64-newlib-x86_64-pnacl-spec',
    ]

PNACL_TOOLCHAIN_TRYBOTS = [
    'nacl-toolchain-linux-pnacl-x86_64',
    'nacl-toolchain-linux-pnacl-x86_32',
    'nacl-toolchain-mac-pnacl-x86_32',
    'nacl-toolchain-win7-pnacl-x86_64',
    ]

TOOLCHAIN_BUILD_TRYBOTS = [
    'nacl-toolchain-precise64-newlib-arm',
    'nacl-toolchain-mac-newlib-arm',
    'nacl-toolchain-win7-newlib-arm',
    ]

def GetPreferredTrySlaves(_, change):

  has_pnacl = False
  has_toolchain_build = False
  has_others = False

  for file in change.AffectedFiles(include_dirs=True):
    if IsFileInDirectories(file.AbsoluteLocalPath(),
                           [os.path.join(NaClTopDir(), 'pnacl')]):
      has_pnacl = True
    elif IsFileInDirectories(file.AbsoluteLocalPath(),
                             [os.path.join(NaClTopDir(), 'toolchain_build')]):
      has_toolchain_build = True
    else:
      has_others = True

  trybots = []
  if has_pnacl:
    trybots += PNACL_TOOLCHAIN_BOTS
  if has_toolchain_build:
    trybots += TOOLCHAIN_BUILD_BOTS
  if has_others:
    trybots += DEFAULT_TRYBOTS

  return trybots
<|endoftext|>


<commit_before>from math import radians, sin, cos, acos, sqrt, pi
import sys

class Cell(object):
    def __init__(self, parameters, symmetry):
        self.cell = parameters
        if parameters:
            assert isinstance(parameters, tuple)
            assert len(parameters) == 6, parameters
            self.a, self.b, self.c = parameters[:3]
            self.alpha, self.beta, self.gamma = parameters[3:]
        if symmetry:
            if ' ' not in symmetry:
                symmetry = add_spaces_to_hm(symmetry)
        self.symmetry = symmetry  # international SG symbol w/ spaces

    def get_volume(self):
        ca = cos(radians(self.alpha))
        cb = cos(radians(self.beta))
        cg = cos(radians(self.gamma))
        return self.a * self.b * self.c * sqrt((1 - ca*ca - cb*cb - cg*cg) +
                                               2 * ca*cb*cg)

    def asu_volume(self, z=None):
        if z is None:
            z = calculate_z_order(self.symmetry)
        return self.get_volume() / z

    def parameters_as_str(self):
        def angle(x):
            if x == 90.: return '90'
            else:        return '%.2f' % x
        return '%.2f, %.2f, %.2f,  %s, %s, %s' % (
                self.a, self.b, self.c,
                angle(self.alpha), angle(self.beta), angle(self.gamma))

    def __str__(self):
        return '%-12s (%s)' % (self.symmetry, self.parameters_as_str())

    # The orthogonalization matrix we use is described in ITfC B p.262:
    # "An alternative mode of orthogonalization, used by the Protein
    # Data Bank and most programs, is to align the a1 axis of the unit
    # cell with the Cartesian X1 axis, and to align the a*3 aixs with the
    # Cartesian X3 axis."
    def get_orth_matrix(self):
        a, b, c = self.a, self.b, self.c
        alpha = radians(self.alpha)
        beta = radians(self.beta)
        gamma = radians(self.gamma)
        alpha_star = acos((cos(gamma)*cos(beta) - cos(alpha)) /
                          (sin(beta)*sin(gamma)))
        return Mat3(a, b*cos(gamma),  c*cos(beta),
                    0, b*sin(gamma), -c*sin(beta)*cos(alpha_star),
                    0, 0,             c*sin(beta)*sin(alpha_star))

    def get_frac_matrix(self):
        return self.get_orth_matrix().inverse()

    def max_shift_in_mapping(self, other):
        trans = self.get_frac_matrix().dot(other.get_orth_matrix())
        return (trans - Mat3.identity()).euclidean_norm()

    # This affects only primitive orthorhombic (P 2x 2x 2x).
    # Convert the "reference" (symmetry-based) settings to the "standard"
    # (cell-based) settings. See the SETTING keyword in POINTLESS.
    def to_standard(self):
        sym = self.symmetry.split()
        if [i[0] for i in sym] == ['P', '2', '2', '2'] and (
                self.a > self.b or self.b > self.c):
            reordered = sorted(zip(self.cell[:3], sym[1:]))
            new_cell, new_symm = zip(*reordered)
            return Cell(new_cell + (90., 90., 90.),
                        symmetry=' '.join((sym[0],) + new_symm))
        return self

    # the opposite of to_symmetry() for oP cells
    def to_reference(self):
        p = self.cell
        if self.symmetry == 'P 21 2 21':
            return Cell((p[0], p[2], p[1]) + p[3:], symmetry='P 21 21 2')
        if self.symmetry == 'P 2 21 21':
            return Cell((p[2], p[1], p[0]) + p[3:], symmetry='P 21 21 2')
        if self.symmetry == 'P 21 2 2':
            return Cell((p[2], p[1], p[0]) + p[3:], symmetry='P 2 2 21')
        if self.symmetry == 'P 2 21 2':
            return Cell((p[0], p[2], p[1]) + p[3:], symmetry='P 2 2 21')
        return self

    # returns symmetry with screw axes removed (changed to rotation axes)
    def unscrewed_symmetry(self):
        return ' '.join(a[0] for a in self.symmetry.split())

class Mat3(object):
    "Matrix 3x3"
    def __init__(self, *args):
        if len(args) == 1:
            self.m = tuple(args[0])
        else:
            self.m = args
        assert len(self.m) == 9

    def __getitem__(self, index):
        return self.m[index]
    def __str__(self):
        return "[%g %g %g; %g %g %g; %g %g %g]" % self.m
    def __repr__(self):
        return "Mat3" + str(self.m)

    def __add__(self, other):
        assert isinstance(other, Mat3)
        return Mat3(a+b for a,b in zip(self.m, other.m))
    def __sub__(self, other):
        assert isinstance(other, Mat3)
        return Mat3(a-b for a,b in zip(self.m, other.m))
    # scalar must be float
    def __mul__(self, scalar):
        assert isinstance(scalar, float)
        return Mat3(a*scalar for a in self.m)

    @staticmethod
    def identity():
        return Mat3(1, 0, 0,
                    0, 1, 0,
                    0, 0, 1)

    def transpose(self):
        m = self.m
        return Mat3(m[0], m[3], m[6],
                    m[1], m[4], m[7],
                    m[2], m[5], m[8])

    def dot(self, other):
        a = self.m
        b = other.m
        return Mat3(sum(a[3*row+i] * b[3*i+col] for i in range(3))
                    for row in range(3)
                    for col in range(3))

    def det(self):
        m = self.m
        return (m[0] * (m[4] * m[8] - m[5] * m[7]) -
                m[1] * (m[3] * m[8] - m[5] * m[6]) +
                m[2] * (m[3] * m[7] - m[4] * m[6]))

    def trace(self):
        m = self.m
        return m[0] + m[4] + m[8]

    def inverse(self):
        d = self.det()
        if d == 0:
            raise ValueError("Matrix is not invertible")
        m = self.m
        return Mat3(( m[4] * m[8] - m[5] * m[7]) / d,
                    (-m[1] * m[8] + m[2] * m[7]) / d,
                    ( m[1] * m[5] - m[2] * m[4]) / d,
                    (-m[3] * m[8] + m[5] * m[6]) / d,
                    ( m[0] * m[8] - m[2] * m[6]) / d,
                    (-m[0] * m[5] + m[2] * m[3]) / d,
                    ( m[3] * m[7] - m[4] * m[6]) / d,
                    (-m[0] * m[7] + m[1] * m[6]) / d,
                    ( m[0] * m[4] - m[1] * m[3]) / d)

    def induced_1norm(self):  # aka 1-norm
        m = self.m
        return max(abs(m[0]) + abs(m[3]) + abs(m[6]),
                   abs(m[1]) + abs(m[4]) + abs(m[7]),
                   abs(m[2]) + abs(m[5]) + abs(m[8]))

    def euclidean_norm(self):  # aka induced 2-norm
        A = self.dot(self.transpose())
        # now get the largest eigenvalue of A (A is symmetric)
        # http://en.wikipedia.org/wiki/Eigenvalue_algorithm#3.C3.973_matrices
        p1 = A[1]**2 + A[2]**2 + A[5]**2
        if p1 == 0:
            eig1 = max(A[0], A[4], A[8])
        else:
            q = A.trace() / 3.
            p2 = (A[0] - q)**2 + (A[4] - q)**2 + (A[8] - q)**2 + 2 * p1
            p = sqrt(p2 / 6.)
            B = (A - Mat3.identity() * q) * (1. / p)
            r = B.det() / 2.
            if r <= -1:
                phi = pi / 3.
            elif r >= 1:
                phi = 0
            else:
                phi = acos(r) / 3.
            eig1 = q + 2 * p * cos(phi)
        return sqrt(eig1)

def calculate_difference(meta1, meta2):
    match = match_symmetry(meta1, meta2)
    # wrong or corrupted file (no CRYST1) is worse than non-matching file
    if match is None:
        return sys.float_info.max
    if match is False:
        return sys.float_info.max / 2
    #return sum(abs(a-b) for a,b in zip(meta1.cell, meta2.cell))
    return meta1.to_standard().max_shift_in_mapping(meta2.to_standard())

# space group utils

def match_symmetry(meta1, meta2):
    if not meta1 or not meta2 or not meta1.symmetry or not meta2.symmetry:
        return None
    def sig(sym):
        first_chars = [a[0] for a in sym.split()]
        s = first_chars[0] + ''.join(sorted(first_chars[1:]))
        if s == 'I112': # I2 is equivalent to C2
            return 'C112'
        # note: I see names such as 'H 3' used in ccp4, never 'R 3'
        if s[0] == 'R':
            s = 'H' + s[1:]
        return s
    return sig(meta1.symmetry) == sig(meta2.symmetry)

_centering_n = {'P': 1, 'A': 2, 'B': 2, 'C': 2, 'I': 2,
                'R': 3, 'H': 3, 'S': 3, 'T': 3, 'F': 4}

_pg_symop = {'1': 1,
             '2': 2, '121': 2,
             '222': 4,
             '4': 4,
             '422': 8,
             '3': 3,
             '32': 6, '312': 6, '321': 6,
             '6': 6,
             '622': 12,
             '23': 12,
             '432': 24,
            }

# This list was generated with cctbx:
#  from cctbx import sgtbx
#  for s in sgtbx.space_group_symbol_iterator():
#    if sgtbx.space_group(s).is_chiral(): print '"%s",' % s.hermann_mauguin()
_hm_symbols = [
"P 1", "P 1 2 1", "P 1 1 2", "P 2 1 1", "P 1 21 1", "P 1 1 21", "P 21 1 1",
"C 1 2 1", "A 1 2 1", "I 1 2 1", "A 1 1 2", "B 1 1 2", "I 1 1 2", "B 2 1 1",
"C 2 1 1", "I 2 1 1", "P 2 2 2", "P 2 2 21", "P 21 2 2", "P 2 21 2",
"P 21 21 2", "P 2 21 21", "P 21 2 21", "P 21 21 21", "C 2 2 21", "A 21 2 2",
"B 2 21 2", "C 2 2 2", "A 2 2 2", "B 2 2 2", "F 2 2 2", "I 2 2 2",
"I 21 21 21", "P 4", "P 41", "P 42", "P 43", "I 4", "I 41", "P 4 2 2",
"P 4 21 2", "P 41 2 2", "P 41 21 2", "P 42 2 2", "P 42 21 2", "P 43 2 2",
"P 43 21 2", "I 4 2 2", "I 41 2 2", "P 3", "P 31", "P 32", "R 3",
"R 3", "P 3 1 2", "P 3 2 1", "P 31 1 2", "P 31 2 1", "P 32 1 2", "P 32 2 1",
"R 3 2", "R 3 2", "P 6", "P 61", "P 65", "P 62", "P 64", "P 63", "P 6 2 2",
"P 61 2 2", "P 65 2 2", "P 62 2 2", "P 64 2 2", "P 63 2 2", "P 2 3", "F 2 3",
"I 2 3", "P 21 3", "I 21 3", "P 4 3 2", "P 42 3 2", "F 4 3 2", "F 41 3 2",
"I 4 3 2", "P 43 3 2", "P 41 3 2", "I 41 3 2"
]

def add_spaces_to_hm(symbol):
    for hm in _hm_symbols:
        if hm.replace(' ', '') == symbol:
            return hm
    return symbol

def calculate_z_order(hm):
    pg = ''.join(a[0] for a in hm.split()[1:])
    return _centering_n[hm[0]] * _pg_symop[pg]

<commit_msg>fix translation of short SG names (without spaces) to full names<commit_after>from math import radians, sin, cos, acos, sqrt, pi
import sys

class Cell(object):
    def __init__(self, parameters, symmetry):
        self.cell = parameters
        if parameters:
            assert isinstance(parameters, tuple)
            assert len(parameters) == 6, parameters
            self.a, self.b, self.c = parameters[:3]
            self.alpha, self.beta, self.gamma = parameters[3:]
        if symmetry in _short_spg_names:
            symmetry = _short_spg_names[symmetry]
        self.symmetry = symmetry  # international SG symbol w/ spaces

    def get_volume(self):
        ca = cos(radians(self.alpha))
        cb = cos(radians(self.beta))
        cg = cos(radians(self.gamma))
        return self.a * self.b * self.c * sqrt((1 - ca*ca - cb*cb - cg*cg) +
                                               2 * ca*cb*cg)

    def asu_volume(self, z=None):
        if z is None:
            z = calculate_z_order(self.symmetry)
        return self.get_volume() / z

    def parameters_as_str(self):
        def angle(x):
            if x == 90.: return '90'
            else:        return '%.2f' % x
        return '%.2f, %.2f, %.2f,  %s, %s, %s' % (
                self.a, self.b, self.c,
                angle(self.alpha), angle(self.beta), angle(self.gamma))

    def __str__(self):
        return '%-12s (%s)' % (self.symmetry, self.parameters_as_str())

    # The orthogonalization matrix we use is described in ITfC B p.262:
    # "An alternative mode of orthogonalization, used by the Protein
    # Data Bank and most programs, is to align the a1 axis of the unit
    # cell with the Cartesian X1 axis, and to align the a*3 aixs with the
    # Cartesian X3 axis."
    def get_orth_matrix(self):
        a, b, c = self.a, self.b, self.c
        alpha = radians(self.alpha)
        beta = radians(self.beta)
        gamma = radians(self.gamma)
        alpha_star = acos((cos(gamma)*cos(beta) - cos(alpha)) /
                          (sin(beta)*sin(gamma)))
        return Mat3(a, b*cos(gamma),  c*cos(beta),
                    0, b*sin(gamma), -c*sin(beta)*cos(alpha_star),
                    0, 0,             c*sin(beta)*sin(alpha_star))

    def get_frac_matrix(self):
        return self.get_orth_matrix().inverse()

    def max_shift_in_mapping(self, other):
        trans = self.get_frac_matrix().dot(other.get_orth_matrix())
        return (trans - Mat3.identity()).euclidean_norm()

    # This affects only primitive orthorhombic (P 2x 2x 2x).
    # Convert the "reference" (symmetry-based) settings to the "standard"
    # (cell-based) settings. See the SETTING keyword in POINTLESS.
    def to_standard(self):
        sym = self.symmetry.split()
        if [i[0] for i in sym] == ['P', '2', '2', '2'] and (
                self.a > self.b or self.b > self.c):
            reordered = sorted(zip(self.cell[:3], sym[1:]))
            new_cell, new_symm = zip(*reordered)
            return Cell(new_cell + (90., 90., 90.),
                        symmetry=' '.join((sym[0],) + new_symm))
        return self

    # the opposite of to_symmetry() for oP cells
    def to_reference(self):
        p = self.cell
        if self.symmetry == 'P 21 2 21':
            return Cell((p[0], p[2], p[1]) + p[3:], symmetry='P 21 21 2')
        if self.symmetry == 'P 2 21 21':
            return Cell((p[2], p[1], p[0]) + p[3:], symmetry='P 21 21 2')
        if self.symmetry == 'P 21 2 2':
            return Cell((p[2], p[1], p[0]) + p[3:], symmetry='P 2 2 21')
        if self.symmetry == 'P 2 21 2':
            return Cell((p[0], p[2], p[1]) + p[3:], symmetry='P 2 2 21')
        return self

    # returns symmetry with screw axes removed (changed to rotation axes)
    def unscrewed_symmetry(self):
        return ' '.join(a[0] for a in self.symmetry.split())

class Mat3(object):
    "Matrix 3x3"
    def __init__(self, *args):
        if len(args) == 1:
            self.m = tuple(args[0])
        else:
            self.m = args
        assert len(self.m) == 9

    def __getitem__(self, index):
        return self.m[index]
    def __str__(self):
        return "[%g %g %g; %g %g %g; %g %g %g]" % self.m
    def __repr__(self):
        return "Mat3" + str(self.m)

    def __add__(self, other):
        assert isinstance(other, Mat3)
        return Mat3(a+b for a,b in zip(self.m, other.m))
    def __sub__(self, other):
        assert isinstance(other, Mat3)
        return Mat3(a-b for a,b in zip(self.m, other.m))
    # scalar must be float
    def __mul__(self, scalar):
        assert isinstance(scalar, float)
        return Mat3(a*scalar for a in self.m)

    @staticmethod
    def identity():
        return Mat3(1, 0, 0,
                    0, 1, 0,
                    0, 0, 1)

    def transpose(self):
        m = self.m
        return Mat3(m[0], m[3], m[6],
                    m[1], m[4], m[7],
                    m[2], m[5], m[8])

    def dot(self, other):
        a = self.m
        b = other.m
        return Mat3(sum(a[3*row+i] * b[3*i+col] for i in range(3))
                    for row in range(3)
                    for col in range(3))

    def det(self):
        m = self.m
        return (m[0] * (m[4] * m[8] - m[5] * m[7]) -
                m[1] * (m[3] * m[8] - m[5] * m[6]) +
                m[2] * (m[3] * m[7] - m[4] * m[6]))

    def trace(self):
        m = self.m
        return m[0] + m[4] + m[8]

    def inverse(self):
        d = self.det()
        if d == 0:
            raise ValueError("Matrix is not invertible")
        m = self.m
        return Mat3(( m[4] * m[8] - m[5] * m[7]) / d,
                    (-m[1] * m[8] + m[2] * m[7]) / d,
                    ( m[1] * m[5] - m[2] * m[4]) / d,
                    (-m[3] * m[8] + m[5] * m[6]) / d,
                    ( m[0] * m[8] - m[2] * m[6]) / d,
                    (-m[0] * m[5] + m[2] * m[3]) / d,
                    ( m[3] * m[7] - m[4] * m[6]) / d,
                    (-m[0] * m[7] + m[1] * m[6]) / d,
                    ( m[0] * m[4] - m[1] * m[3]) / d)

    def induced_1norm(self):  # aka 1-norm
        m = self.m
        return max(abs(m[0]) + abs(m[3]) + abs(m[6]),
                   abs(m[1]) + abs(m[4]) + abs(m[7]),
                   abs(m[2]) + abs(m[5]) + abs(m[8]))

    def euclidean_norm(self):  # aka induced 2-norm
        A = self.dot(self.transpose())
        # now get the largest eigenvalue of A (A is symmetric)
        # http://en.wikipedia.org/wiki/Eigenvalue_algorithm#3.C3.973_matrices
        p1 = A[1]**2 + A[2]**2 + A[5]**2
        if p1 == 0:
            eig1 = max(A[0], A[4], A[8])
        else:
            q = A.trace() / 3.
            p2 = (A[0] - q)**2 + (A[4] - q)**2 + (A[8] - q)**2 + 2 * p1
            p = sqrt(p2 / 6.)
            B = (A - Mat3.identity() * q) * (1. / p)
            r = B.det() / 2.
            if r <= -1:
                phi = pi / 3.
            elif r >= 1:
                phi = 0
            else:
                phi = acos(r) / 3.
            eig1 = q + 2 * p * cos(phi)
        return sqrt(eig1)

def calculate_difference(meta1, meta2):
    match = match_symmetry(meta1, meta2)
    # wrong or corrupted file (no CRYST1) is worse than non-matching file
    if match is None:
        return sys.float_info.max
    if match is False:
        return sys.float_info.max / 2
    #return sum(abs(a-b) for a,b in zip(meta1.cell, meta2.cell))
    return meta1.to_standard().max_shift_in_mapping(meta2.to_standard())

# space group utils

def match_symmetry(meta1, meta2):
    if not meta1 or not meta2 or not meta1.symmetry or not meta2.symmetry:
        return None
    def sig(sym):
        first_chars = [a[0] for a in sym.split()]
        s = first_chars[0] + ''.join(sorted(first_chars[1:]))
        if s == 'I112': # I2 is equivalent to C2
            return 'C112'
        # note: I see names such as 'H 3' used in ccp4, never 'R 3'
        if s[0] == 'R':
            s = 'H' + s[1:]
        return s
    return sig(meta1.symmetry) == sig(meta2.symmetry)

_centering_n = {'P': 1, 'A': 2, 'B': 2, 'C': 2, 'I': 2,
                'R': 3, 'H': 3, 'S': 3, 'T': 3, 'F': 4}

_pg_symop = {'1': 1,
             '2': 2, '121': 2,
             '222': 4,
             '4': 4,
             '422': 8,
             '3': 3,
             '32': 6, '312': 6, '321': 6,
             '6': 6,
             '622': 12,
             '23': 12,
             '432': 24,
            }

# the list of space group names extracted from symop.lib with this script:
#
# import shlex
# from cctbx import sgtbx
# for line in open('symop.lib'):
#     if line and not line[0].isspace():
#         fields = line.partition('!')[0].split(None, 6)
#         spacegroups = shlex.split(fields[-1])
#         sgs = sgtbx.space_group_symbols(int(fields[0]) % 1000)
#         sg = sgtbx.space_group(sgs.hall())
#         if sg.is_chiral():
#             print '"%s": "%s",' % (fields[3], spacegroups[0])
_short_spg_names = {
"P1": "P 1", "P2": "P 1 2 1", "P21": "P 1 21 1", "C2": "C 1 2 1",
"P222": "P 2 2 2", "P2221": "P 2 2 21", "P21212": "P 21 21 2",
"P212121": "P 21 21 21", "C2221": "C 2 2 21", "C222": "C 2 2 2",
"F222": "F 2 2 2", "I222": "I 2 2 2", "I212121": "I 21 21 21",
"P4": "P 4", "P41": "P 41", "P42": "P 42", "P43": "P 43", "I4": "I 4",
"I41": "I 41", "P422": "P 4 2 2", "P4212": "P 4 21 2", "P4122": "P 41 2 2",
"P41212": "P 41 21 2", "P4222": "P 42 2 2", "P42212": "P 42 21 2",
"P4322": "P 43 2 2", "P43212": "P 43 21 2", "I422": "I 4 2 2",
"I4122": "I 41 2 2", "P3": "P 3", "P31": "P 31", "P32": "P 32", "H3": "H 3",
"R3": "R 3", "P312": "P 3 1 2", "P321": "P 3 2 1", "P3112": "P 31 1 2",
"P3121": "P 31 2 1", "P3212": "P 32 1 2", "P3221": "P 32 2 1", "H32": "H 3 2",
"R32": "R 3 2", "P6": "P 6", "P61": "P 61", "P65": "P 65", "P62": "P 62",
"P64": "P 64", "P63": "P 63", "P622": "P 6 2 2", "P6122": "P 61 2 2",
"P6522": "P 65 2 2", "P6222": "P 62 2 2", "P6422": "P 64 2 2",
"P6322": "P 63 2 2", "P23": "P 2 3", "F23": "F 2 3", "I23": "I 2 3",
"P213": "P 21 3", "I213": "I 21 3", "P432": "P 4 3 2", "P4232": "P 42 3 2",
"F432": "F 4 3 2", "F4132": "F 41 3 2", "I432": "I 4 3 2", "P4332": "P 43 3 2",
"P4132": "P 41 3 2", "I4132": "I 41 3 2", "P112": "P 1 1 2",
"P1121": "P 1 1 21", "B2": "B 1 1 2", "A2": "A 1 2 1", "C21": "C 1 21 1",
"I2": "I 1 2 1", "I21": "I 1 21 1", "P2122": "P 21 2 2", "P2212": "P 2 21 2",
"P21212a": "P 21 21 2 (a)", "P21221": "P 21 2 21", "P22121": "P 2 21 21",
"C2221a": "C 2 2 21a)", "C222a": "C 2 2 2a", "F222a": "F 2 2 2a",
"I222a": "I 2 2 2a", "P42212a": "P 42 21 2a", "I23a": "I 2 3a",
}

def calculate_z_order(hm):
    pg = ''.join(a[0] for a in hm.split()[1:])
    return _centering_n[hm[0]] * _pg_symop[pg]

<|endoftext|>


<commit_before><commit_msg>nail in pylint newline coffin for temp<commit_after><|endoftext|>


<commit_before><commit_msg>cf cli v6.26 chaged output from `cf target`<commit_after><|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django import forms
from amcat.scripts.output import commandline
from amcat.scripts import scriptmanager
from amcat.scripts.script import Script
import argparse
import sys
from amcat.tools import toolkit, amcatlogging
import types

import logging; log = logging.getLogger(__name__)
#logging.basicConfig(format='[%(asctime)s] [%(name)s] %(message)s', level=logging.DEBUG)
#logging.getLogger('django.db.backends').setLevel(logging.ERROR)

###############################################################
##         Aux method for CLI invocation                     ##
###############################################################

def get_main_script(module):
    if type(module) == type(''):
	module_name = module
	module = __import__(module_name)
	for part in module_name.split('.')[1:]:
	    module = getattr(module, part)
    for name in dir(module):
	obj = getattr(module, name)
	if (isinstance(obj, (type, types.ClassType)) and
	    issubclass(obj, Script)):
	    return obj
    raise Exception("Cannot find Script module in %r" % module)

def run_cli(cls=None, handle_output=None):
    """Handle command line interface invocation of this script"""
    amcatlogging.setup()

    if cls is None:
	mod = toolkit.getCallingModule()
	cls = get_main_script(mod)

    if handle_output is None:
	handle_output = cls.output_type != None

    parser = argument_parser_from_script(cls)
    args = parser.parse_args()
    options = args.__dict__
    instance = cls(options)

    input = None
    if cls.input_type in (file, str, unicode):
	input = sys.stdin
    if cls.input_type in (str, unicode):
	input = input.read()
    if cls.input_type == unicode:
	input = input.decode('latin-1')

    out = instance.run(input)

    if handle_output:
        out = handleOutput(out, instance.output_type)
    return out

def handleOutput(out, output_type):
    cls = scriptmanager.findScript(output_type, str)
    if not cls:
        print(out)
    else:
        print cls().run(out)

def argument_parser_from_script(script_class):
    """Create an argparse.ArgumentParser from a Script class object"""
    parser = argparse.ArgumentParser(description=script_class.__doc__)
    used_prefixes = set("-h") # keep track of -X options used; -h -> help
    if script_class.options_form:
        form = script_class.options_form()
        for name, field in form.fields.items():
           add_argument_from_field(parser, name, field)
    #parser.add_argument('--output', help='set output type', default='print')
    return parser

def add_argument_from_field(parser, name, field):
    """
    Call parser.add_argument with the appropriate parameters
    to add this django.forms.Field to the argparse.ArgumentParsers parser.
    """
    # set options name to lowercase variant, remember name for storage
    optname = name.lower()
    if field.required and field.initial is None: # positional argument
        argname = [optname]
    else: # optional argument
        argname = ["--"+optname]
        prefix = "-" + optname[0]
        if prefix not in parser._option_string_actions:
            argname.append("-"+optname[0])
    # determine help text from label, help_text, and default
    helpfields = [x for x in [field.label, field.help_text] if x]
    if field.initial is not None: helpfields.append("Default: {field.initial}".format(**locals()))
    help = ". ".join(helpfields)
    # set type OR action_const as extra arguments depending on type
    args = {} # flexible parameters to add_argument
    if isinstance(field, forms.ModelMultipleChoiceField) or isinstance(field, forms.MultipleChoiceField):
        args["nargs"] = '+'
        argname = ['--' + optname] # problem: now the field appears to be optional, but at least it knows which values belong to this argument
    if isinstance(field, forms.BooleanField) and field.initial is not None:
        args["action"] = "store_const"
        args["const"] = not field.initial
    else:
        args["type"] = argument_type_from_field(field)
    parser.add_argument(*argname, help = help, default=field.initial, dest=name, **args)

_FIELD_MAP = {
    forms.IntegerField : int,
    forms.BooleanField : bool,
    }
def argument_type_from_field(field):
    """Get the proper python type to parse a command line option"""
    for (ftype, type) in _FIELD_MAP.iteritems():
        if isinstance(field, ftype): return type
    return str
<commit_msg>Fixed bug causing ValueError<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django import forms
from amcat.scripts import scriptmanager
from amcat.scripts.script import Script
import argparse
import sys
from amcat.tools import toolkit, amcatlogging
import types

import logging; log = logging.getLogger(__name__)
#logging.basicConfig(format='[%(asctime)s] [%(name)s] %(message)s', level=logging.DEBUG)
#logging.getLogger('django.db.backends').setLevel(logging.ERROR)

###############################################################
##         Aux method for CLI invocation                     ##
###############################################################

def get_main_script(module):
    if type(module) == type(''):
	module_name = module
	module = __import__(module_name)
	for part in module_name.split('.')[1:]:
	    module = getattr(module, part)
    for name in dir(module):
	obj = getattr(module, name)
	if (isinstance(obj, (type, types.ClassType)) and
	    issubclass(obj, Script)):
	    return obj
    raise Exception("Cannot find Script module in %r" % module)

def run_cli(cls=None, handle_output=None):
    """Handle command line interface invocation of this script"""
    amcatlogging.setup()

    if cls is None:
	mod = toolkit.getCallingModule()
	cls = get_main_script(mod)

    if handle_output is None:
	handle_output = cls.output_type != None

    parser = argument_parser_from_script(cls)
    args = parser.parse_args()
    options = args.__dict__
    instance = cls(options)

    input = None
    if cls.input_type in (file, str, unicode):
	input = sys.stdin
    if cls.input_type in (str, unicode):
	input = input.read()
    if cls.input_type == unicode:
	input = input.decode('latin-1')

    out = instance.run(input)

    if handle_output:
        out = handleOutput(out, instance.output_type)
    return out

def handleOutput(out, output_type):
    cls = scriptmanager.findScript(output_type, str)
    if not cls:
        print(out)
    else:
        print cls().run(out)

def argument_parser_from_script(script_class):
    """Create an argparse.ArgumentParser from a Script class object"""
    parser = argparse.ArgumentParser(description=script_class.__doc__)
    used_prefixes = set("-h") # keep track of -X options used; -h -> help
    if script_class.options_form:
        form = script_class.options_form()
        for name, field in form.fields.items():
           add_argument_from_field(parser, name, field)
    #parser.add_argument('--output', help='set output type', default='print')
    return parser

def add_argument_from_field(parser, name, field):
    """
    Call parser.add_argument with the appropriate parameters
    to add this django.forms.Field to the argparse.ArgumentParsers parser.
    """
    # set options name to lowercase variant, remember name for storage
    optname = name.lower()
    if field.required and field.initial is None: # positional argument
        argname = [optname]
    else: # optional argument
        argname = ["--"+optname]
        prefix = "-" + optname[0]
        if prefix not in parser._option_string_actions:
            argname.append("-"+optname[0])
    # determine help text from label, help_text, and default
    helpfields = [x for x in [field.label, field.help_text] if x]
    if field.initial is not None: helpfields.append("Default: {field.initial}".format(**locals()))
    help = ". ".join(helpfields)
    # set type OR action_const as extra arguments depending on type
    args = {} # flexible parameters to add_argument
    if isinstance(field, forms.ModelMultipleChoiceField) or isinstance(field, forms.MultipleChoiceField):
        args["nargs"] = '+'
        argname = ['--' + optname] # problem: now the field appears to be optional, but at least it knows which values belong to this argument
    if isinstance(field, forms.BooleanField) and field.initial is not None:
        args["action"] = "store_const"
        args["const"] = not field.initial
    else:
        args["type"] = argument_type_from_field(field)

    parser.add_argument(*argname, help = help, default=field.initial, metavar=name, **args)

_FIELD_MAP = {
    forms.IntegerField : int,
    forms.BooleanField : bool,
    }
def argument_type_from_field(field):
    """Get the proper python type to parse a command line option"""
    for (ftype, type) in _FIELD_MAP.iteritems():
        if isinstance(field, ftype): return type
    return str
<|endoftext|>


<commit_before>
import asyncio
from functools import partial
import json
import os
import random
import string
import time

from aiohttp import web
from asyncio_redis import Connection, ZScoreBoundary

import bleach

BASE_DIR = os.path.dirname(__file__)

RATE_LIMIT_DURATION = 60
RATE_LIMIT = 10

def make_key(*args):
    return':'.join(args)

# For bleach
def linkify_external(attrs, new=False):
    attrs['target'] = '_blank'
    return attrs

strip_tags = partial(bleach.clean, tags=[], strip=True)

@asyncio.coroutine
def post_message(request, message, mode, queue=None, **data):
    if queue is None:
        queue = make_key(request.match_info['channel'], 'channel')

    nick = yield from get_nick(request)

    data.setdefault('message', message)
    data.setdefault('sender', nick)

    content = json.dumps(data)
    yield from request.conn.publish(queue, json.dumps([mode, content]))

# Nick handling
@asyncio.coroutine
def get_nicks(request):
    key = make_key(request.match_info['channel'], 'nick', '*')
    keys = yield from request.conn.keys_aslist(key)
    if keys:
        vals = yield from request.conn.mget_aslist(keys)
        return {
            k: v
            for k, v in zip(vals, keys)
        }
    return {}

@asyncio.coroutine
def get_nick(request):
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    nick = yield from request.conn.get(key)
    if nick is None:
        nick = yield from set_nick(request, request.tag[:8])
    else:
        yield from request.conn.expire(key, 90)
    return nick

@asyncio.coroutine
def set_nick(request, name):
    name = strip_tags(name)
    nicks = yield from get_nicks(request)
    if name in nicks:
        raise ValueError('Nick in use!')
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    yield from request.conn.set(key, name, expire=90)
    return name

# Topic handling
@asyncio.coroutine
def set_topic(request, topic):
    key = make_key(request.match_info['channel'], 'topic')
    yield from request.conn.set(key, topic)

@asyncio.coroutine
def get_topic(request):
    key = make_key(request.match_info['channel'], 'topic')
    topic = yield from request.conn.get(key)
    return topic

# Request handlers
@asyncio.coroutine
def index(request):
    return web.Response(
        body=open(os.path.join(BASE_DIR, 'index.html'), 'rb').read()
    )

class SseResponse(web.StreamResponse):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super().__init__(*args, **kwargs)
        self.headers[web.hdrs.CONTENT_TYPE] = 'text/event-stream; charset=utf-8'
        self.headers['Cache-Control'] = 'no-cache'

    def write_eof(self):
        request = self.request

        subscriber = yield from request.conn.start_subscribe()

        yield from subscriber.subscribe([
            make_key(request.match_info['channel'], 'channel'),
            make_key(request.tag, 'private'),
        ])

        while True:
            msg = yield from subscriber.next_published()
            mode, data = json.loads(msg.value)
            self.write('event: {}\n'.format(mode).encode('utf-8'))
            for line in data.splitlines():
                self.write('data: {}\n'.format(line).encode('utf-8'))
            self.write('\n'.encode('utf-8'))

@asyncio.coroutine
def listen(request):
    if 'text/event-stream' not in request.headers['ACCEPT']:
        return web.http.HTTPNotAcceptable()

    nick = yield from get_nick(request)

    yield from post_message(request, '{} connected.'.format(nick), 'join',
                            sender='Notice')

    return SseResponse(request)

@asyncio.coroutine
def chatter(request):
    yield from request.post()

    mode = request.POST.get('mode', 'message')
    msg = request.POST.get('message', '')
    msg = bleach.linkify(strip_tags(msg), callbacks=[linkify_external])

    nick = yield from get_nick(request)

    if mode == 'nick' and msg:
        try:
            new_nick = yield from set_nick(request, msg)
        except ValueError:
            yield from post_message(request, 'Nick in use!', 'alert',
                                    sender='Notice')
        else:
            yield from post_message(
                request,
                '{} is now known as {}'.format(nick, new_nick),
                mode='nick',
                sender='Notice'
            )

    elif mode == 'names':
        nicks = yield from get_nicks(request)
        yield from post_message(request, list(nicks.keys()), 'names')

    elif mode == 'msg':
        target = request.POST['target']
        nicks = yield from get_nicks(request)
        _, _, target_tag = nicks[target].split(':')
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(target_tag, 'private'))
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(request.tag, 'private'))

    elif mode in ['message', 'action']:
        yield from post_message(request, msg, mode)

    elif mode == 'topic':
        if msg:
            yield from set_topic(request, msg)
        topic = yield from get_topic(request)
        yield from post_message(request, topic, 'topic')

    return web.Response(body=b'')

@asyncio.coroutine
def cookie_middleware(app, handler):
    @asyncio.coroutine
    def middleware(request):
        tag = request.cookies.get('chatterbox', None)
        request.tag = tag or ''.join(random.choice(string.ascii_letters)
                                     for x in range(16))

        request.conn = yield from Connection.create(host='localhost', port=6379)

        # Rate limit
        key = make_key(request.tag, 'rated')
        now = time.time()
        yield from request.conn.zadd(key, {str(int(now)): now})
        yield from request.conn.expireat(key, int(now) + RATE_LIMIT_DURATION)
        yield from request.conn.zremrangebyscore(
            key,
            ZScoreBoundary('-inf'),
            ZScoreBoundary(now - RATE_LIMIT_DURATION)
        )
        size = yield from request.conn.zcard(key)

        if size > RATE_LIMIT:
            response = web.Response(body=b'', status=429)
        else:
            # Call handler
            response = yield from handler(request)
        # Set cookie
        if tag is None:
            response.set_cookie('chatterbox', request.tag)
        return response
    return middleware

if __name__ == '__main__':
    app = web.Application(middlewares=[cookie_middleware])
    app.router.add_route('GET', '/', index)
    app.router.add_static('/static/', os.path.join(BASE_DIR, 'static'))
    app.router.add_route('GET', '/{channel}/', listen)
    app.router.add_route('POST', '/{channel}/', chatter)

    loop = asyncio.get_event_loop()
    f = loop.create_server(app.make_handler(), '0.0.0.0', 8080)
    srv = loop.run_until_complete(f)
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
<commit_msg>[Fixed #3] Close redis connections<commit_after>
import asyncio
from functools import partial
import json
import os
import random
import string
import time

from aiohttp import web
from asyncio_redis import Connection, ZScoreBoundary

import bleach

BASE_DIR = os.path.dirname(__file__)

RATE_LIMIT_DURATION = 60
RATE_LIMIT = 10

def make_key(*args):
    return':'.join(args)

# For bleach
def linkify_external(attrs, new=False):
    attrs['target'] = '_blank'
    return attrs

strip_tags = partial(bleach.clean, tags=[], strip=True)

@asyncio.coroutine
def post_message(request, message, mode, queue=None, **data):
    if queue is None:
        queue = make_key(request.match_info['channel'], 'channel')

    nick = yield from get_nick(request)

    data.setdefault('message', message)
    data.setdefault('sender', nick)

    content = json.dumps(data)
    yield from request.conn.publish(queue, json.dumps([mode, content]))

# Nick handling
@asyncio.coroutine
def get_nicks(request):
    key = make_key(request.match_info['channel'], 'nick', '*')
    keys = yield from request.conn.keys_aslist(key)
    if keys:
        vals = yield from request.conn.mget_aslist(keys)
        return {
            k: v
            for k, v in zip(vals, keys)
        }
    return {}

@asyncio.coroutine
def get_nick(request):
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    nick = yield from request.conn.get(key)
    if nick is None:
        nick = yield from set_nick(request, request.tag[:8])
    else:
        yield from request.conn.expire(key, 90)
    return nick

@asyncio.coroutine
def set_nick(request, name):
    name = strip_tags(name)
    nicks = yield from get_nicks(request)
    if name in nicks:
        raise ValueError('Nick in use!')
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    yield from request.conn.set(key, name, expire=90)
    return name

# Topic handling
@asyncio.coroutine
def set_topic(request, topic):
    key = make_key(request.match_info['channel'], 'topic')
    yield from request.conn.set(key, topic)

@asyncio.coroutine
def get_topic(request):
    key = make_key(request.match_info['channel'], 'topic')
    topic = yield from request.conn.get(key)
    return topic

# Request handlers
@asyncio.coroutine
def index(request):
    return web.Response(
        body=open(os.path.join(BASE_DIR, 'index.html'), 'rb').read()
    )

class SseResponse(web.StreamResponse):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super().__init__(*args, **kwargs)
        self.headers[web.hdrs.CONTENT_TYPE] = 'text/event-stream; charset=utf-8'
        self.headers['Cache-Control'] = 'no-cache'

    def write_eof(self):
        request = self.request

        subscriber = yield from request.conn.start_subscribe()

        yield from subscriber.subscribe([
            make_key(request.match_info['channel'], 'channel'),
            make_key(request.tag, 'private'),
        ])

        while True:
            msg = yield from subscriber.next_published()
            mode, data = json.loads(msg.value)
            self.write('event: {}\n'.format(mode).encode('utf-8'))
            for line in data.splitlines():
                self.write('data: {}\n'.format(line).encode('utf-8'))
            self.write('\n'.encode('utf-8'))

        self.conn.close()

@asyncio.coroutine
def listen(request):
    if 'text/event-stream' not in request.headers['ACCEPT']:
        return web.http.HTTPNotAcceptable()

    nick = yield from get_nick(request)

    yield from post_message(request, '{} connected.'.format(nick), 'join',
                            sender='Notice')

    return SseResponse(request)

@asyncio.coroutine
def chatter(request):
    yield from request.post()

    mode = request.POST.get('mode', 'message')
    msg = request.POST.get('message', '')
    msg = bleach.linkify(strip_tags(msg), callbacks=[linkify_external])

    nick = yield from get_nick(request)

    if mode == 'nick' and msg:
        try:
            new_nick = yield from set_nick(request, msg)
        except ValueError:
            yield from post_message(request, 'Nick in use!', 'alert',
                                    sender='Notice')
        else:
            yield from post_message(
                request,
                '{} is now known as {}'.format(nick, new_nick),
                mode='nick',
                sender='Notice'
            )

    elif mode == 'names':
        nicks = yield from get_nicks(request)
        yield from post_message(request, list(nicks.keys()), 'names')

    elif mode == 'msg':
        target = request.POST['target']
        nicks = yield from get_nicks(request)
        _, _, target_tag = nicks[target].split(':')
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(target_tag, 'private'))
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(request.tag, 'private'))

    elif mode in ['message', 'action']:
        yield from post_message(request, msg, mode)

    elif mode == 'topic':
        if msg:
            yield from set_topic(request, msg)
        topic = yield from get_topic(request)
        yield from post_message(request, topic, 'topic')

    return web.Response(body=b'')

@asyncio.coroutine
def cookie_middleware(app, handler):
    @asyncio.coroutine
    def middleware(request):
        tag = request.cookies.get('chatterbox', None)
        request.tag = tag or ''.join(random.choice(string.ascii_letters)
                                     for x in range(16))

        request.conn = yield from Connection.create(host='localhost', port=6379)

        # Rate limit
        key = make_key(request.tag, 'rated')
        now = time.time()
        yield from request.conn.zadd(key, {str(int(now)): now})
        yield from request.conn.expireat(key, int(now) + RATE_LIMIT_DURATION)
        yield from request.conn.zremrangebyscore(
            key,
            ZScoreBoundary('-inf'),
            ZScoreBoundary(now - RATE_LIMIT_DURATION)
        )
        size = yield from request.conn.zcard(key)

        if size > RATE_LIMIT:
            response = web.Response(body=b'', status=429)
        else:
            # Call handler
            response = yield from handler(request)
        # Set cookie
        if tag is None:
            response.set_cookie('chatterbox', request.tag)
        if not isinstance(response, SseResponse):
            request.conn.close()
        return response
    return middleware

if __name__ == '__main__':
    app = web.Application(middlewares=[cookie_middleware])
    app.router.add_route('GET', '/', index)
    app.router.add_static('/static/', os.path.join(BASE_DIR, 'static'))
    app.router.add_route('GET', '/{channel}/', listen)
    app.router.add_route('POST', '/{channel}/', chatter)

    loop = asyncio.get_event_loop()
    f = loop.create_server(app.make_handler(), '0.0.0.0', 8080)
    srv = loop.run_until_complete(f)
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>
import asyncio
from functools import partial
import json
import mimetypes
import os
import random
import string
import time

from aiohttp import web
from asyncio_redis import Connection, ZScoreBoundary

import bleach

BASE_DIR = os.path.dirname(__file__)

RATE_LIMIT_DURATION = 60
RATE_LIMIT = 10

def make_key(*args):
    return':'.join(args)

# For bleach
def linkify_external(attrs, new=False):
    attrs['target'] = '_blank'
    return attrs

strip_tags = partial(bleach.clean, tags=[], strip=True)

@asyncio.coroutine
def post_message(request, message, mode, queue=None, **data):
    if queue is None:
        queue = make_key(request.match_info['channel'], 'channel')

    nick = yield from get_nick(request)

    data.setdefault('message', message)
    data.setdefault('sender', nick)

    content = json.dumps(data)
    yield from request.conn.publish(queue, json.dumps([mode, content]))

# Nick handling
@asyncio.coroutine
def get_nicks(request):
    key = make_key(request.match_info['channel'], 'nick', '*')
    keys = yield from request.conn.keys_aslist(key)
    if keys:
        vals = yield from request.conn.mget_aslist(keys)
        return {
            k: v
            for k, v in zip(vals, keys)
        }
    return {}

@asyncio.coroutine
def get_nick(request):
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    nick = yield from request.conn.get(key)
    if nick is None:
        nick = yield from set_nick(request, request.tag[:8])
    else:
        yield from request.conn.expire(key, 90)
    return nick

@asyncio.coroutine
def set_nick(request, name):
    name = strip_tags(name)
    nicks = yield from get_nicks(request)
    if name in nicks:
        raise ValueError('Nick in use!')
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    yield from request.conn.set(key, name, expire=90)
    return name

# Topic handling
@asyncio.coroutine
def set_topic(request, topic):
    key = make_key(request.match_info['channel'], 'topic')
    yield from request.conn.set(key, topic)

@asyncio.coroutine
def get_topic(request):
    key = make_key(request.match_info['channel'], 'topic')
    topic = yield from request.conn.get(key)
    return topic

# Request handlers
@asyncio.coroutine
def index(request):
    return web.Response(
        body=open(os.path.join(BASE_DIR, 'index.html'), 'rb').read()
    )

@asyncio.coroutine
def static(request):
    path = os.path.abspath(
        os.path.join(BASE_DIR, 'static', request.match_info['path'])
    )
    if not path.startswith(BASE_DIR):
        return web.HTTPNotFound()

    content_type, encoding = mimetypes.guess_type(path)
    return web.Response(body=open(path, 'rb').read(), content_type=content_type)

class SseResponse(web.StreamResponse):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super().__init__(*args, **kwargs)
        self.headers[web.hdrs.CONTENT_TYPE] = 'text/event-stream; charset=utf-8'
        self.headers['Cache-Control'] = 'no-cache'

    def write_eof(self):
        request = self.request

        subscriber = yield from request.conn.start_subscribe()

        yield from subscriber.subscribe([
            make_key(request.match_info['channel'], 'channel'),
            make_key(request.tag, 'private'),
        ])

        while True:
            msg = yield from subscriber.next_published()
            mode, data = json.loads(msg.value)
            self.write('event: {}\n'.format(mode).encode('utf-8'))
            for line in data.splitlines():
                self.write('data: {}\n'.format(line).encode('utf-8'))
            self.write('\n'.encode('utf-8'))

@asyncio.coroutine
def listen(request):
    if 'text/event-stream' not in request.headers['ACCEPT']:
        return web.http.HTTPNotAcceptable()

    nick = yield from get_nick(request)

    yield from post_message(request, '{} connected.'.format(nick), 'join',
                            sender='Notice')

    return SseResponse(request)

@asyncio.coroutine
def chatter(request):
    yield from request.post()

    mode = request.POST.get('mode', 'message')
    msg = request.POST.get('message', '')
    msg = bleach.linkify(strip_tags(msg), callbacks=[linkify_external])

    nick = yield from get_nick(request)

    if mode == 'nick' and msg:
        try:
            new_nick = yield from set_nick(request, msg)
        except ValueError:
            yield from post_message(request, 'Nick in use!', 'alert',
                                    sender='Notice')
        else:
            yield from post_message(
                request,
                '{} is now known as {}'.format(nick, new_nick),
                mode='nick',
                sender='Notice'
            )

    elif mode == 'names':
        nicks = yield from get_nicks(request)
        yield from post_message(request, list(nicks.keys()), 'names')

    elif mode == 'msg':
        target = request.POST['target']
        nicks = yield from get_nicks(request)
        _, _, target_tag = nicks[target].split(':')
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(target_tag, 'private'))
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(request.tag, 'private'))

    elif mode in ['message', 'action']:
        yield from post_message(request, msg, mode)

    elif mode == 'topic':
        if msg:
            yield from set_topic(request, msg)
        topic = yield from get_topic(request)
        yield from post_message(request, topic, 'topic')

    return web.Response(body=b'')

@asyncio.coroutine
def cookie_middleware(app, handler):
    @asyncio.coroutine
    def middleware(request):
        tag = request.cookies.get('chatterbox', None)
        request.tag = tag or ''.join(random.choice(string.ascii_letters)
                                     for x in range(16))

        request.conn = yield from Connection.create(host='localhost', port=6379)

        # Rate limit
        key = make_key(request.tag, 'rated')
        now = time.time()
        yield from request.conn.zadd(key, {str(int(now)): now})
        yield from request.conn.expireat(key, int(now) + RATE_LIMIT_DURATION)
        yield from request.conn.zremrangebyscore(
            key,
            ZScoreBoundary('-inf'),
            ZScoreBoundary(now - RATE_LIMIT_DURATION)
        )
        size = yield from request.conn.zcard(key)

        if size > RATE_LIMIT:
            response = web.Response(body=b'', status_code=429)
        else:
            # Call handler
            response = yield from handler(request)
        # Set cookie
        if tag is None:
            response.set_cookie('chatterbox', request.tag)
        return response
    return middleware

if __name__ == '__main__':
    app = web.Application(middlewares=[cookie_middleware])
    app.router.add_route('GET', '/', index)
    app.router.add_route('GET', '/static/{path:.+}', static)
    app.router.add_route('GET', '/{channel}/', listen)
    app.router.add_route('POST', '/{channel}/', chatter)

    loop = asyncio.get_event_loop()
    f = loop.create_server(app.make_handler(), '0.0.0.0', 8080)
    srv = loop.run_until_complete(f)
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
<commit_msg>Replace static view with built in static route<commit_after>
import asyncio
from functools import partial
import json
import mimetypes
import os
import random
import string
import time

from aiohttp import web
from asyncio_redis import Connection, ZScoreBoundary

import bleach

BASE_DIR = os.path.dirname(__file__)

RATE_LIMIT_DURATION = 60
RATE_LIMIT = 10

def make_key(*args):
    return':'.join(args)

# For bleach
def linkify_external(attrs, new=False):
    attrs['target'] = '_blank'
    return attrs

strip_tags = partial(bleach.clean, tags=[], strip=True)

@asyncio.coroutine
def post_message(request, message, mode, queue=None, **data):
    if queue is None:
        queue = make_key(request.match_info['channel'], 'channel')

    nick = yield from get_nick(request)

    data.setdefault('message', message)
    data.setdefault('sender', nick)

    content = json.dumps(data)
    yield from request.conn.publish(queue, json.dumps([mode, content]))

# Nick handling
@asyncio.coroutine
def get_nicks(request):
    key = make_key(request.match_info['channel'], 'nick', '*')
    keys = yield from request.conn.keys_aslist(key)
    if keys:
        vals = yield from request.conn.mget_aslist(keys)
        return {
            k: v
            for k, v in zip(vals, keys)
        }
    return {}

@asyncio.coroutine
def get_nick(request):
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    nick = yield from request.conn.get(key)
    if nick is None:
        nick = yield from set_nick(request, request.tag[:8])
    else:
        yield from request.conn.expire(key, 90)
    return nick

@asyncio.coroutine
def set_nick(request, name):
    name = strip_tags(name)
    nicks = yield from get_nicks(request)
    if name in nicks:
        raise ValueError('Nick in use!')
    key = make_key(request.match_info['channel'], 'nick', request.tag)
    yield from request.conn.set(key, name, expire=90)
    return name

# Topic handling
@asyncio.coroutine
def set_topic(request, topic):
    key = make_key(request.match_info['channel'], 'topic')
    yield from request.conn.set(key, topic)

@asyncio.coroutine
def get_topic(request):
    key = make_key(request.match_info['channel'], 'topic')
    topic = yield from request.conn.get(key)
    return topic

# Request handlers
@asyncio.coroutine
def index(request):
    return web.Response(
        body=open(os.path.join(BASE_DIR, 'index.html'), 'rb').read()
    )

class SseResponse(web.StreamResponse):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super().__init__(*args, **kwargs)
        self.headers[web.hdrs.CONTENT_TYPE] = 'text/event-stream; charset=utf-8'
        self.headers['Cache-Control'] = 'no-cache'

    def write_eof(self):
        request = self.request

        subscriber = yield from request.conn.start_subscribe()

        yield from subscriber.subscribe([
            make_key(request.match_info['channel'], 'channel'),
            make_key(request.tag, 'private'),
        ])

        while True:
            msg = yield from subscriber.next_published()
            mode, data = json.loads(msg.value)
            self.write('event: {}\n'.format(mode).encode('utf-8'))
            for line in data.splitlines():
                self.write('data: {}\n'.format(line).encode('utf-8'))
            self.write('\n'.encode('utf-8'))

@asyncio.coroutine
def listen(request):
    if 'text/event-stream' not in request.headers['ACCEPT']:
        return web.http.HTTPNotAcceptable()

    nick = yield from get_nick(request)

    yield from post_message(request, '{} connected.'.format(nick), 'join',
                            sender='Notice')

    return SseResponse(request)

@asyncio.coroutine
def chatter(request):
    yield from request.post()

    mode = request.POST.get('mode', 'message')
    msg = request.POST.get('message', '')
    msg = bleach.linkify(strip_tags(msg), callbacks=[linkify_external])

    nick = yield from get_nick(request)

    if mode == 'nick' and msg:
        try:
            new_nick = yield from set_nick(request, msg)
        except ValueError:
            yield from post_message(request, 'Nick in use!', 'alert',
                                    sender='Notice')
        else:
            yield from post_message(
                request,
                '{} is now known as {}'.format(nick, new_nick),
                mode='nick',
                sender='Notice'
            )

    elif mode == 'names':
        nicks = yield from get_nicks(request)
        yield from post_message(request, list(nicks.keys()), 'names')

    elif mode == 'msg':
        target = request.POST['target']
        nicks = yield from get_nicks(request)
        _, _, target_tag = nicks[target].split(':')
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(target_tag, 'private'))
        yield from post_message(request, msg, 'msg', target=target,
                                queue=make_key(request.tag, 'private'))

    elif mode in ['message', 'action']:
        yield from post_message(request, msg, mode)

    elif mode == 'topic':
        if msg:
            yield from set_topic(request, msg)
        topic = yield from get_topic(request)
        yield from post_message(request, topic, 'topic')

    return web.Response(body=b'')

@asyncio.coroutine
def cookie_middleware(app, handler):
    @asyncio.coroutine
    def middleware(request):
        tag = request.cookies.get('chatterbox', None)
        request.tag = tag or ''.join(random.choice(string.ascii_letters)
                                     for x in range(16))

        request.conn = yield from Connection.create(host='localhost', port=6379)

        # Rate limit
        key = make_key(request.tag, 'rated')
        now = time.time()
        yield from request.conn.zadd(key, {str(int(now)): now})
        yield from request.conn.expireat(key, int(now) + RATE_LIMIT_DURATION)
        yield from request.conn.zremrangebyscore(
            key,
            ZScoreBoundary('-inf'),
            ZScoreBoundary(now - RATE_LIMIT_DURATION)
        )
        size = yield from request.conn.zcard(key)

        if size > RATE_LIMIT:
            response = web.Response(body=b'', status_code=429)
        else:
            # Call handler
            response = yield from handler(request)
        # Set cookie
        if tag is None:
            response.set_cookie('chatterbox', request.tag)
        return response
    return middleware

if __name__ == '__main__':
    app = web.Application(middlewares=[cookie_middleware])
    app.router.add_route('GET', '/', index)
    app.router.add_static('/static/', os.path.join(BASE_DIR, 'static'))
    app.router.add_route('GET', '/{channel}/', listen)
    app.router.add_route('POST', '/{channel}/', chatter)

    loop = asyncio.get_event_loop()
    f = loop.create_server(app.make_handler(), '0.0.0.0', 8080)
    srv = loop.run_until_complete(f)
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
<|endoftext|>