<commit_before>"""
Defines the Sumatra version control interface for Mercurial.

Classes
-------

MercurialWorkingCopy
MercurialRepository

Functions
---------

may_have_working_copy() - determine whether a .hg subdirectory exists at a given
                          path
get_working_copy()      - return a MercurialWorkingCopy object for a given path
get_repository()        - return a MercurialRepository object for a given URL.
"""

from mercurial import hg, ui, patch
try:
    from mercurial.error import RepoError
except ImportError:
    from mercurial.repo import RepoError
try:
    from mercurial.cmdutil import findrepo
except ImportError:
    from mercurial.dispatch import _findrepo as findrepo
import os
import binascii
from base import VersionControlError

from base import Repository, WorkingCopy

def may_have_working_copy(path=None):
    path = path or os.getcwd()
    return bool(findrepo(path))

def get_working_copy(path=None):
    if may_have_working_copy(path):
        return MercurialWorkingCopy(path)
    else:
        raise VersionControlError("No Mercurial working copy found at %s" % path)

def get_repository(url):
    return MercurialRepository(url)

class MercurialWorkingCopy(WorkingCopy):

    def __init__(self, path=None):
        WorkingCopy.__init__(self)
        self.path = path or os.getcwd()
        self.repository = MercurialRepository(self.path)
        self.repository.working_copy = self

    def current_version(self):
        if hasattr(self.repository._repository, 'workingctx'): # handle different versions of Mercurial
            ctx = self.repository._repository.workingctx().parents()[0]
        else:
            ctx = self.repository._repository.parents()[0]
        return binascii.hexlify(ctx.node()[:6])

    def use_version(self, version):
        assert not self.has_changed()
        hg.clean(self.repository._repository, version)

    def use_latest_version(self):
        # any changes to the working copy are retained/merged in
        hg.update(self.repository._repository, 'tip')

    def status(self):
        modified, added, removed, deleted, unknown, ignored, clean = self.repository._repository.status() #unknown=True)
        return {'modified': modified, 'removed': removed,
                'deleted': deleted, 'unknown': unknown}

    def has_changed(self):
        status = self.status()
        changed = False
        for st in 'modified', 'removed', 'deleted':
            if status[st]:
                changed = True
                break
        return changed

    def diff(self):
        """Difference between working copy and repository."""
        opts = patch.mdiff.diffopts(nodates=True)
        diff = patch.diff(self.repository._repository, opts=opts)
        return "".join(diff)

class MercurialRepository(Repository):

    def __init__(self, url):
        Repository.__init__(self, url)
        self._ui = ui.ui()  # get a ui object
        try:
            self._repository = hg.repository(self._ui, url)
        # need to add a check that this actually is a Mercurial repository
        except (RepoError, Exception), err:
            raise VersionControlError("%s" % err)
        self.working_copy = None

    def checkout(self, path="."):
        """Clone a repository."""
        path = os.path.abspath(path)
        if self.url == path:
            # update
            hg.update(self._repository, None)
        else:
            # can't clone into an existing directory, so we create
            # an empty repository and then do a pull and update
            local_repos = hg.repository(self._ui, path, create=True)
            local_repos.pull(self._repository)
            hg.update(local_repos, None)
        self.working_copy = MercurialWorkingCopy(path)

    def __getstate__(self):
        """For pickling"""
        return {'url': self.url, 'wc': self.working_copy}

    def __setstate__(self, state):
        """For unpickling"""
        self.__init__(state['url'])
        self.working_copy = state['wc']
<commit_msg>`_mercurial.get_working_copy()` now accepts any path within the working copy<commit_after>"""
Defines the Sumatra version control interface for Mercurial.

Classes
-------

MercurialWorkingCopy
MercurialRepository

Functions
---------

may_have_working_copy() - determine whether a .hg subdirectory exists at a given
                          path
get_working_copy()      - return a MercurialWorkingCopy object for a given path
get_repository()        - return a MercurialRepository object for a given URL.
"""

from mercurial import hg, ui, patch
try:
    from mercurial.error import RepoError
except ImportError:
    from mercurial.repo import RepoError
try:
    from mercurial.cmdutil import findrepo
except ImportError:
    from mercurial.dispatch import _findrepo as findrepo
import os
import binascii
from base import VersionControlError

from base import Repository, WorkingCopy

def may_have_working_copy(path=None):
    path = path or os.getcwd()
    return bool(findrepo(path))

def get_working_copy(path=None):
    repo_dir = findrepo(path or os.getcwd())
    if repo_dir:
        return MercurialWorkingCopy(repo_dir)
    else:
        raise VersionControlError("No Mercurial working copy found at %s" % path)

def get_repository(url):
    return MercurialRepository(url)

class MercurialWorkingCopy(WorkingCopy):

    def __init__(self, path=None):
        WorkingCopy.__init__(self)
        self.path = path or os.getcwd()
        self.repository = MercurialRepository(self.path)
        self.repository.working_copy = self

    def current_version(self):
        if hasattr(self.repository._repository, 'workingctx'): # handle different versions of Mercurial
            ctx = self.repository._repository.workingctx().parents()[0]
        else:
            ctx = self.repository._repository.parents()[0]
        return binascii.hexlify(ctx.node()[:6])

    def use_version(self, version):
        assert not self.has_changed()
        hg.clean(self.repository._repository, version)

    def use_latest_version(self):
        # any changes to the working copy are retained/merged in
        hg.update(self.repository._repository, 'tip')

    def status(self):
        modified, added, removed, deleted, unknown, ignored, clean = self.repository._repository.status() #unknown=True)
        return {'modified': modified, 'removed': removed,
                'deleted': deleted, 'unknown': unknown}

    def has_changed(self):
        status = self.status()
        changed = False
        for st in 'modified', 'removed', 'deleted':
            if status[st]:
                changed = True
                break
        return changed

    def diff(self):
        """Difference between working copy and repository."""
        opts = patch.mdiff.diffopts(nodates=True)
        diff = patch.diff(self.repository._repository, opts=opts)
        return "".join(diff)

class MercurialRepository(Repository):

    def __init__(self, url):
        Repository.__init__(self, url)
        self._ui = ui.ui()  # get a ui object
        try:
            self._repository = hg.repository(self._ui, url)
        # need to add a check that this actually is a Mercurial repository
        except (RepoError, Exception), err:
            raise VersionControlError("%s" % err)
        self.working_copy = None

    def checkout(self, path="."):
        """Clone a repository."""
        path = os.path.abspath(path)
        if self.url == path:
            # update
            hg.update(self._repository, None)
        else:
            # can't clone into an existing directory, so we create
            # an empty repository and then do a pull and update
            local_repos = hg.repository(self._ui, path, create=True)
            local_repos.pull(self._repository)
            hg.update(local_repos, None)
        self.working_copy = MercurialWorkingCopy(path)

    def __getstate__(self):
        """For pickling"""
        return {'url': self.url, 'wc': self.working_copy}

    def __setstate__(self, state):
        """For unpickling"""
        self.__init__(state['url'])
        self.working_copy = state['wc']
<|endoftext|>


<commit_before>from unittest import TestCase
import math
import numpy
from chainer      import cuda, Variable
from chainer.cuda import to_cpu, to_gpu, GPUArray
from chainer.gradient_check import assert_allclose, numerical_grad
from chainer.functions import sigmoid_cross_entropy
from chainer.testing import attr

if cuda.available:
    cuda.init()

class TestSigmoidCrossEntropy(TestCase):
    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (4, 3)).astype(numpy.float32)
        self.t = numpy.random.randint(0, 2, (4, 3)).astype(numpy.int32)

    def check_forward(self, x_data, t_data, use_cudnn=True):
        x_val = Variable(x_data)
        t_val = Variable(t_data)
        loss = sigmoid_cross_entropy(x_val, t_val, use_cudnn)
        loss_value = float(to_cpu(loss.data))

        # Compute expected value
        loss_expect = 0
        for i in xrange(self.x.shape[0]):
            for j in xrange(self.x.shape[1]):
                xd, td = self.x[i, j], self.t[i, j]
                xdd = 1 if self.x[i, j] >= 0 else 0
                loss_expect -= xd * (td - xdd) - math.log(1 + math.exp(-numpy.abs(xd)))
        loss_expect /= self.t.shape[0]
        self.assertAlmostEqual(loss_expect, loss_value, places=5)

    def test_forward_cpu(self):
        self.check_forward(self.x, self.t)

    @attr.cudnn
    def test_forward_gpu(self):
        self.check_forward(to_gpu(self.x), to_gpu(self.t))

    @attr.gpu
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(to_gpu(self.x), to_gpu(self.t), False)

    def check_backward(self, x_data, t_data, use_cudnn=True):
        x = Variable(x_data)
        t = Variable(t_data)
        loss = sigmoid_cross_entropy(x, t, use_cudnn)
        loss.backward()
        self.assertEqual(None, t.grad)

        func = loss.creator
        f = lambda: func.forward((x.data, t.data))
        gx, = numerical_grad(f, (x.data,), (1,), eps=0.01)

        assert_allclose(gx, x.grad)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.t)

    @attr.cudnn
    def test_backward_gpu(self):
        self.check_backward(to_gpu(self.x), to_gpu(self.t))

    @attr.gpu
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(to_gpu(self.x), to_gpu(self.t), False)
<commit_msg>Use six.moves.range instead of xrange<commit_after>from unittest import TestCase
import math
import numpy
from six.moves import range
from chainer      import cuda, Variable
from chainer.cuda import to_cpu, to_gpu, GPUArray
from chainer.gradient_check import assert_allclose, numerical_grad
from chainer.functions import sigmoid_cross_entropy
from chainer.testing import attr

if cuda.available:
    cuda.init()

class TestSigmoidCrossEntropy(TestCase):
    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (4, 3)).astype(numpy.float32)
        self.t = numpy.random.randint(0, 2, (4, 3)).astype(numpy.int32)

    def check_forward(self, x_data, t_data, use_cudnn=True):
        x_val = Variable(x_data)
        t_val = Variable(t_data)
        loss = sigmoid_cross_entropy(x_val, t_val, use_cudnn)
        loss_value = float(to_cpu(loss.data))

        # Compute expected value
        loss_expect = 0
        for i in range(self.x.shape[0]):
            for j in range(self.x.shape[1]):
                xd, td = self.x[i, j], self.t[i, j]
                xdd = 1 if self.x[i, j] >= 0 else 0
                loss_expect -= xd * (td - xdd) - math.log(1 + math.exp(-numpy.abs(xd)))
        loss_expect /= self.t.shape[0]
        self.assertAlmostEqual(loss_expect, loss_value, places=5)

    def test_forward_cpu(self):
        self.check_forward(self.x, self.t)

    @attr.cudnn
    def test_forward_gpu(self):
        self.check_forward(to_gpu(self.x), to_gpu(self.t))

    @attr.gpu
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(to_gpu(self.x), to_gpu(self.t), False)

    def check_backward(self, x_data, t_data, use_cudnn=True):
        x = Variable(x_data)
        t = Variable(t_data)
        loss = sigmoid_cross_entropy(x, t, use_cudnn)
        loss.backward()
        self.assertEqual(None, t.grad)

        func = loss.creator
        f = lambda: func.forward((x.data, t.data))
        gx, = numerical_grad(f, (x.data,), (1,), eps=0.01)

        assert_allclose(gx, x.grad)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.t)

    @attr.cudnn
    def test_backward_gpu(self):
        self.check_backward(to_gpu(self.x), to_gpu(self.t))

    @attr.gpu
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(to_gpu(self.x), to_gpu(self.t), False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2012 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Starts up an Impala cluster (ImpalaD + State Store) with the specified number of
# ImpalaD instances. Each ImpalaD runs on a different port allowing this to be run
# on a single machine.
import os
import sys
from time import sleep, time
from optparse import OptionParser

# Options
parser = OptionParser()
parser.add_option("-s", "--cluster_size", type="int", dest="cluster_size", default=3,
                  help="Size of the cluster (number of impalad instances to start).")
parser.add_option("--build_type", dest="build_type", default= 'debug',
                  help="Build type to use - debug / release")
parser.add_option("--impalad_args", dest="impalad_args", default="",
                  help="Additional arguments to pass to each Impalad during startup")
parser.add_option("--state_store_args", dest="state_store_args", default="",
                  help="Additional arguments to pass to State Store during startup")
parser.add_option("--catalogd_args", dest="catalogd_args", default="",
                  help="Additional arguments to pass to the Catalog Service at startup")
parser.add_option("--kill", "--kill_only", dest="kill_only", action="store_true",
                  default=False, help="Instead of starting the cluster, just kill all"\
                  " the running impalads and the statestored.")
parser.add_option("--force_kill", dest="force_kill", action="store_true", default=False,
                  help="Force kill impalad and statestore processes.")
parser.add_option("--in-process", dest="inprocess", action="store_true", default=False,
                  help="Start all Impala backends and state store in a single process.")
parser.add_option("--log_dir", dest="log_dir", default="/tmp",
                  help="Directory to store output logs to.")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                  help="Prints all output to stderr/stdout.")
parser.add_option("--wait_for_cluster", dest="wait_for_cluster", action="store_true",
                  default=False, help="Wait until the cluster is ready to accept "\
                  "queries before returning.")
parser.add_option("--log_level", type="int", dest="log_level", default=1,
                   help="Set the impalad backend logging level")

options, args = parser.parse_args()

IMPALA_HOME = os.environ['IMPALA_HOME']
KNOWN_BUILD_TYPES = ['debug', 'release']
IMPALAD_PATH = os.path.join(IMPALA_HOME,
    'bin/start-impalad.sh -build_type=%s' % options.build_type)
STATE_STORE_PATH = os.path.join(IMPALA_HOME, 'be/build',
    options.build_type, 'statestore/statestored')
CATALOGD_PATH = os.path.join(IMPALA_HOME,
    'bin/start-catalogd.sh -build_type=%s' % options.build_type)
MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"

IMPALA_SHELL = os.path.join(IMPALA_HOME, 'bin/impala-shell.sh')
IMPALAD_PORTS = ("-beeswax_port=%d -hs2_port=%d  -be_port=%d "
                 "-state_store_subscriber_port=%d -webserver_port=%d")
BE_LOGGING_ARGS = "-log_filename=%s -log_dir=%s -v=%s -logbufsecs=5"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240

def exec_impala_process(cmd, args, stderr_log_file_path):
  redirect_output = str()
  if options.verbose:
    args += ' -logtostderr=1'
  else:
    redirect_output = "1>%s" % stderr_log_file_path
  cmd = '%s %s %s 2>&1 &' % (cmd, args, redirect_output)
  os.system(cmd)

def kill_all(force=False):
  kill_cmd = "killall"
  if force:
    kill_cmd += " -9"
  os.system("%s catalogd" % kill_cmd)
  os.system("%s mini-impala-cluster" % kill_cmd)
  os.system("%s impalad" % kill_cmd)
  os.system("%s statestored" % kill_cmd)
  sleep(1)

def start_statestore():
  print "Starting State Store logging to %s/statestored.INFO" % options.log_dir
  stderr_log_file_path = os.path.join(options.log_dir, "statestore-error.log")
  args = "%s %s" % (build_impalad_logging_args(0, "statestored"),
                    options.state_store_args)
  exec_impala_process(STATE_STORE_PATH, args, stderr_log_file_path)

def start_catalogd():
  print "Starting Catalog Service logging to %s/catalogd.INFO" % options.log_dir
  stderr_log_file_path = os.path.join(options.log_dir, "catalogd-error.log")
  args = "%s %s" % (build_impalad_logging_args(0, "catalogd"),
                    options.catalogd_args)
  exec_impala_process(CATALOGD_PATH, args, stderr_log_file_path)

def start_mini_impala_cluster(cluster_size):
  print ("Starting in-process Impala Cluster logging "
         "to %s/mini-impala-cluster.INFO" % options.log_dir)
  args = "-num_backends=%s" % (cluster_size)
  args += ' ' + build_impalad_logging_args(0, 'mini-impala-cluster')
  stderr_log_file_path = os.path.join(options.log_dir, 'mini-impala-cluster-error.log')
  exec_impala_process(MINI_IMPALA_CLUSTER_PATH, args, stderr_log_file_path)

def build_impalad_port_args(instance_num):
  BASE_BEESWAX_PORT = 21000
  BASE_HS2_PORT = 21050
  BASE_BE_PORT = 22000
  BASE_STATE_STORE_SUBSCRIBER_PORT = 23000
  BASE_WEBSERVER_PORT = 25000
  return IMPALAD_PORTS % (BASE_BEESWAX_PORT + instance_num, BASE_HS2_PORT + instance_num,
                          BASE_BE_PORT + instance_num,
                          BASE_STATE_STORE_SUBSCRIBER_PORT + instance_num,
                          BASE_WEBSERVER_PORT +instance_num)

def build_impalad_logging_args(instance_num, service_name):
  log_file_path = os.path.join(options.log_dir, "%s.INFO" % service_name)
  return BE_LOGGING_ARGS % (service_name, options.log_dir, options.log_level)

def start_impalad_instances(cluster_size):
  # Start each impalad instance and optionally redirect the output to a log file.
  for i in range(options.cluster_size):
    if i == 0:
      # The first impalad always logs to impalad.INFO
      service_name = "impalad"
    else:
      service_name = "impalad_node%s" % i
    args = build_impalad_logging_args(i, service_name)
    args += ' ' + build_impalad_port_args(i) +  ' ' + options.impalad_args
    stderr_log_file_path = os.path.join(options.log_dir, '%s-error.log' % service_name)
    exec_impala_process(IMPALAD_PATH, args, stderr_log_file_path)

def wait_for_impala_process_count(impala_cluster, retries=3):
  """Checks that the desired number of impalad/statestored processes are running.

  Refresh until the number running impalad/statestored processes reaches the expected
  number based on CLUSTER_SIZE, or the retry limit is hit. Failing this, raise a
  RuntimeError.
  """
  for i in range(retries):
    if len(impala_cluster.impalads) < options.cluster_size or \
        not impala_cluster.statestored or not impala_cluster.catalogd:
          sleep(2)
          impala_cluster.refresh()
  msg = str()
  if len(impala_cluster.impalads) < options.cluster_size:
    impalads_found = len(impala_cluster.impalads)
    msg += "Expected %d impalad(s), only %d found\n" %\
        (options.cluster_size, impalads_found)
  if not impala_cluster.statestored:
    msg += "statestored failed to start.\n"
  if not impala_cluster.catalogd:
    msg += "catalogd failed to start.\n"
  if msg:
    raise RuntimeError, msg

def wait_for_cluster_web(timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS):
  """Checks if the cluster is "ready"

  A cluster is deemed "ready" if:
    - All backends are registered with the statestore.
    - Each impalad knows about all other impalads.
  This information is retrieved by querying the statestore debug webpage
  and each individual impalad's metrics webpage.
  """
  impala_cluster = ImpalaCluster()
  # impalad processes may take a while to come up.
  wait_for_impala_process_count(impala_cluster)
  for impalad in impala_cluster.impalads:
    impalad.service.wait_for_num_known_live_backends(options.cluster_size, interval=2)
    wait_for_catalog(impalad, timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS)

def wait_for_catalog(impalad, timeout_in_seconds):
  """Waits for the impalad catalog to become ready"""
  start_time = time()
  catalog_ready = False
  while (time() - start_time < timeout_in_seconds and not catalog_ready):
    try:
      num_dbs = impalad.service.get_metric_value('catalog.num-databases')
      num_tbls = impalad.service.get_metric_value('catalog.num-tables')
      catalog_ready = impalad.service.get_metric_value('catalog.ready')
      print 'Waiting for Catalog... Status: %s DBs / %s tables (ready=%s)' %\
          (num_dbs, num_tbls, catalog_ready)
    except Exception, e:
      print e
    sleep(1)
  if not catalog_ready:
    raise RuntimeError, 'Catalog was not initialized in expected time period.'

def wait_for_cluster_cmdline(timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS):
  """Checks if the cluster is "ready" by executing a simple query in a loop"""
  start_time = time()
  while os.system('%s -i localhost:21000 -q "%s"' %  (IMPALA_SHELL, 'select 1')) != 0:
    if time() - timeout_in_seconds > start_time:
      raise RuntimeError, 'Cluster did not start within %d seconds' % timeout_in_seconds
    print 'Cluster not yet available. Sleeping...'
    sleep(2)

if __name__ == "__main__":
  if options.kill_only:
    kill_all(force=options.force_kill)
    sys.exit(0)

  if options.build_type not in KNOWN_BUILD_TYPES:
    print 'Invalid build type %s' % options.build_type
    print 'Valid values: %s' % ', '.join(KNOWN_BUILD_TYPES)
    sys.exit(1)

  if options.cluster_size <= 0:
    print 'Please specify a cluster size > 0'
    sys.exit(1)

  # Kill existing processes.
  kill_all(force=options.force_kill)

  try:
    import json
    wait_for_cluster = wait_for_cluster_web
  except ImportError:
    print "json module not found, checking for cluster startup through the command-line"
    wait_for_cluster = wait_for_cluster_cmdline

  # If ImpalaCluster cannot be imported, fall back to the command-line to check
  # whether impalads/statestore are up.
  try:
    from tests.common.impala_cluster import ImpalaCluster
    # Make sure the processes have been killed. We loop till we can't detect a single
    # impalad or a statestore process.
    impala_cluster = ImpalaCluster()
    while len(impala_cluster.impalads) != 0 or impala_cluster.statestored or\
          impala_cluster.catalogd:
      impala_cluster.refresh()
  except ImportError:
    print 'ImpalaCluster module not found.'
    wait_for_cluster = wait_for_cluster_cmdline

  if options.inprocess:
    # The statestore and the impalads start in the same process. Additionally,
    # the statestore does not have a debug webpage.
    start_mini_impala_cluster(options.cluster_size)
    wait_for_cluster_cmdline()
  else:
    try:
      start_statestore()
      start_catalogd()
      start_impalad_instances(options.cluster_size)
      wait_for_cluster()
    except Exception, e:
      print 'Error starting cluster: %s' % e
      sys.exit(1)

  print 'Impala Cluster Running with %d nodes.' % options.cluster_size
<commit_msg>Add option to start-impala-cluster to only restart impalad<commit_after>#!/usr/bin/env python
# Copyright 2012 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Starts up an Impala cluster (ImpalaD + State Store) with the specified number of
# ImpalaD instances. Each ImpalaD runs on a different port allowing this to be run
# on a single machine.
import os
import sys
from time import sleep, time
from optparse import OptionParser

# Options
parser = OptionParser()
parser.add_option("-s", "--cluster_size", type="int", dest="cluster_size", default=3,
                  help="Size of the cluster (number of impalad instances to start).")
parser.add_option("--build_type", dest="build_type", default= 'debug',
                  help="Build type to use - debug / release")
parser.add_option("--impalad_args", dest="impalad_args", default="",
                  help="Additional arguments to pass to each Impalad during startup")
parser.add_option("--state_store_args", dest="state_store_args", default="",
                  help="Additional arguments to pass to State Store during startup")
parser.add_option("--catalogd_args", dest="catalogd_args", default="",
                  help="Additional arguments to pass to the Catalog Service at startup")
parser.add_option("--kill", "--kill_only", dest="kill_only", action="store_true",
                  default=False, help="Instead of starting the cluster, just kill all"\
                  " the running impalads and the statestored.")
parser.add_option("--force_kill", dest="force_kill", action="store_true", default=False,
                  help="Force kill impalad and statestore processes.")
parser.add_option("-r", "--restart_impalad_only", dest="restart_impalad_only",
                  action="store_true", default=False,
                  help="Restarts only the impalad processes")
parser.add_option("--in-process", dest="inprocess", action="store_true", default=False,
                  help="Start all Impala backends and state store in a single process.")
parser.add_option("--log_dir", dest="log_dir", default="/tmp",
                  help="Directory to store output logs to.")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                  help="Prints all output to stderr/stdout.")
parser.add_option("--wait_for_cluster", dest="wait_for_cluster", action="store_true",
                  default=False, help="Wait until the cluster is ready to accept "\
                  "queries before returning.")
parser.add_option("--log_level", type="int", dest="log_level", default=1,
                   help="Set the impalad backend logging level")

options, args = parser.parse_args()

IMPALA_HOME = os.environ['IMPALA_HOME']
KNOWN_BUILD_TYPES = ['debug', 'release']
IMPALAD_PATH = os.path.join(IMPALA_HOME,
    'bin/start-impalad.sh -build_type=%s' % options.build_type)
STATE_STORE_PATH = os.path.join(IMPALA_HOME, 'be/build',
    options.build_type, 'statestore/statestored')
CATALOGD_PATH = os.path.join(IMPALA_HOME,
    'bin/start-catalogd.sh -build_type=%s' % options.build_type)
MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"

IMPALA_SHELL = os.path.join(IMPALA_HOME, 'bin/impala-shell.sh')
IMPALAD_PORTS = ("-beeswax_port=%d -hs2_port=%d  -be_port=%d "
                 "-state_store_subscriber_port=%d -webserver_port=%d")
BE_LOGGING_ARGS = "-log_filename=%s -log_dir=%s -v=%s -logbufsecs=5"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240

def exec_impala_process(cmd, args, stderr_log_file_path):
  redirect_output = str()
  if options.verbose:
    args += ' -logtostderr=1'
  else:
    redirect_output = "1>%s" % stderr_log_file_path
  cmd = '%s %s %s 2>&1 &' % (cmd, args, redirect_output)
  os.system(cmd)

def kill_cluster_processes(force=False):
  kill_matching_processes('catalogd')
  kill_matching_processes('impalad')
  kill_matching_processes('statestored')
  kill_matching_processes('mini-impala-cluster')

def kill_matching_processes(binary_name, force=False):
  """Kills all processes with the given binary name"""
  # -w = Wait for processes to die.
  kill_cmd = "killall -w"
  if force: kill_cmd += " -9"
  os.system("%s %s" % (kill_cmd, binary_name))

def start_statestore():
  print "Starting State Store logging to %s/statestored.INFO" % options.log_dir
  stderr_log_file_path = os.path.join(options.log_dir, "statestore-error.log")
  args = "%s %s" % (build_impalad_logging_args(0, "statestored"),
                    options.state_store_args)
  exec_impala_process(STATE_STORE_PATH, args, stderr_log_file_path)

def start_catalogd():
  print "Starting Catalog Service logging to %s/catalogd.INFO" % options.log_dir
  stderr_log_file_path = os.path.join(options.log_dir, "catalogd-error.log")
  args = "%s %s" % (build_impalad_logging_args(0, "catalogd"),
                    options.catalogd_args)
  exec_impala_process(CATALOGD_PATH, args, stderr_log_file_path)

def start_mini_impala_cluster(cluster_size):
  print ("Starting in-process Impala Cluster logging "
         "to %s/mini-impala-cluster.INFO" % options.log_dir)
  args = "-num_backends=%s" % (cluster_size)
  args += ' ' + build_impalad_logging_args(0, 'mini-impala-cluster')
  stderr_log_file_path = os.path.join(options.log_dir, 'mini-impala-cluster-error.log')
  exec_impala_process(MINI_IMPALA_CLUSTER_PATH, args, stderr_log_file_path)

def build_impalad_port_args(instance_num):
  BASE_BEESWAX_PORT = 21000
  BASE_HS2_PORT = 21050
  BASE_BE_PORT = 22000
  BASE_STATE_STORE_SUBSCRIBER_PORT = 23000
  BASE_WEBSERVER_PORT = 25000
  return IMPALAD_PORTS % (BASE_BEESWAX_PORT + instance_num, BASE_HS2_PORT + instance_num,
                          BASE_BE_PORT + instance_num,
                          BASE_STATE_STORE_SUBSCRIBER_PORT + instance_num,
                          BASE_WEBSERVER_PORT +instance_num)

def build_impalad_logging_args(instance_num, service_name):
  log_file_path = os.path.join(options.log_dir, "%s.INFO" % service_name)
  return BE_LOGGING_ARGS % (service_name, options.log_dir, options.log_level)

def start_impalad_instances(cluster_size):
  # Start each impalad instance and optionally redirect the output to a log file.
  for i in range(options.cluster_size):
    if i == 0:
      # The first impalad always logs to impalad.INFO
      service_name = "impalad"
    else:
      service_name = "impalad_node%s" % i
    args = build_impalad_logging_args(i, service_name)
    args += ' ' + build_impalad_port_args(i) +  ' ' + options.impalad_args
    stderr_log_file_path = os.path.join(options.log_dir, '%s-error.log' % service_name)
    exec_impala_process(IMPALAD_PATH, args, stderr_log_file_path)

def wait_for_impala_process_count(impala_cluster, retries=3):
  """Checks that the desired number of impalad/statestored processes are running.

  Refresh until the number running impalad/statestored processes reaches the expected
  number based on CLUSTER_SIZE, or the retry limit is hit. Failing this, raise a
  RuntimeError.
  """
  for i in range(retries):
    if len(impala_cluster.impalads) < options.cluster_size or \
        not impala_cluster.statestored or not impala_cluster.catalogd:
          sleep(2)
          impala_cluster.refresh()
  msg = str()
  if len(impala_cluster.impalads) < options.cluster_size:
    impalads_found = len(impala_cluster.impalads)
    msg += "Expected %d impalad(s), only %d found\n" %\
        (options.cluster_size, impalads_found)
  if not impala_cluster.statestored:
    msg += "statestored failed to start.\n"
  if not impala_cluster.catalogd:
    msg += "catalogd failed to start.\n"
  if msg:
    raise RuntimeError, msg

def wait_for_cluster_web(timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS):
  """Checks if the cluster is "ready"

  A cluster is deemed "ready" if:
    - All backends are registered with the statestore.
    - Each impalad knows about all other impalads.
  This information is retrieved by querying the statestore debug webpage
  and each individual impalad's metrics webpage.
  """
  impala_cluster = ImpalaCluster()
  # impalad processes may take a while to come up.
  wait_for_impala_process_count(impala_cluster)
  for impalad in impala_cluster.impalads:
    impalad.service.wait_for_num_known_live_backends(options.cluster_size, interval=2)
    wait_for_catalog(impalad, timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS)

def wait_for_catalog(impalad, timeout_in_seconds):
  """Waits for the impalad catalog to become ready"""
  start_time = time()
  catalog_ready = False
  while (time() - start_time < timeout_in_seconds and not catalog_ready):
    try:
      num_dbs = impalad.service.get_metric_value('catalog.num-databases')
      num_tbls = impalad.service.get_metric_value('catalog.num-tables')
      catalog_ready = impalad.service.get_metric_value('catalog.ready')
      print 'Waiting for Catalog... Status: %s DBs / %s tables (ready=%s)' %\
          (num_dbs, num_tbls, catalog_ready)
    except Exception, e:
      print e
    sleep(1)
  if not catalog_ready:
    raise RuntimeError, 'Catalog was not initialized in expected time period.'

def wait_for_cluster_cmdline(timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS):
  """Checks if the cluster is "ready" by executing a simple query in a loop"""
  start_time = time()
  while os.system('%s -i localhost:21000 -q "%s"' %  (IMPALA_SHELL, 'select 1')) != 0:
    if time() - timeout_in_seconds > start_time:
      raise RuntimeError, 'Cluster did not start within %d seconds' % timeout_in_seconds
    print 'Cluster not yet available. Sleeping...'
    sleep(2)

if __name__ == "__main__":
  if options.kill_only:
    kill_cluster_processes(force=options.force_kill)
    sys.exit(0)

  if options.build_type not in KNOWN_BUILD_TYPES:
    print 'Invalid build type %s' % options.build_type
    print 'Valid values: %s' % ', '.join(KNOWN_BUILD_TYPES)
    sys.exit(1)

  if options.cluster_size <= 0:
    print 'Please specify a cluster size > 0'
    sys.exit(1)

  # Kill existing cluster processes based on the current configuration.
  if options.restart_impalad_only:
    if options.inprocess:
      print 'Cannot perform individual component restarts using an in-process cluster'
      sys.exit(1)
    kill_matching_processes('impalad', force=options.force_kill)
  else:
    kill_cluster_processes(force=options.force_kill)

  try:
    import json
    wait_for_cluster = wait_for_cluster_web
  except ImportError:
    print "json module not found, checking for cluster startup through the command-line"
    wait_for_cluster = wait_for_cluster_cmdline

  # If ImpalaCluster cannot be imported, fall back to the command-line to check
  # whether impalads/statestore are up.
  try:
    from tests.common.impala_cluster import ImpalaCluster
    if options.restart_impalad_only:
      impala_cluster = ImpalaCluster()
      if not impala_cluster.statestored or not impala_cluster.catalogd:
        print 'No running statestored or catalogd detected. Restarting entire cluster.'
        options.restart_impalad_only = False
  except ImportError:
    print 'ImpalaCluster module not found.'
    # TODO: Update this code path to work similar to the ImpalaCluster code path when
    # restarting only impalad processes. Specifically, we should do a full cluster
    # restart if either the statestored or catalogd processes are down, even if
    # restart_only_impalad=True.
    wait_for_cluster = wait_for_cluster_cmdline

  if options.inprocess:
    # The statestore and the impalads start in the same process.
    start_mini_impala_cluster(options.cluster_size)
    wait_for_cluster_cmdline()
  else:
    try:
      if not options.restart_impalad_only:
        start_statestore()
        start_catalogd()
      start_impalad_instances(options.cluster_size)
      wait_for_cluster()
    except Exception, e:
      print 'Error starting cluster: %s' % e
      sys.exit(1)

  print 'Impala Cluster Running with %d nodes.' % options.cluster_size
<|endoftext|>


<commit_before>from wq.db.rest.serializers import ModelSerializer
from rest_framework import serializers
from rest_framework.utils import model_meta
from wq.db.rest.models import get_ct, get_object_id
from wq.db.rest import compat as html
from .models import NaturalKeyModel

class AttachmentListSerializer(serializers.ListSerializer):
    def to_representation(self, data):
        data = super(AttachmentListSerializer, self).to_representation(data)
        if self.parent:
            for i, row in enumerate(data):
                row['@index'] = i
        return data

class TypedAttachmentListSerializer(AttachmentListSerializer):
    def get_value(self, dictionary):
        # Handle attachments that are submitted together with their parent
        value = super(TypedAttachmentListSerializer, self).get_value(
            dictionary
        )
        if not isinstance(value, list):
            return []
        for i, row in enumerate(value):
            empty = True
            if isinstance(row, dict):
                for key, val in row.items():
                    if key == self.child.Meta.type_field:
                        continue
                    elif val:
                        empty = False
            if empty:
                value[i] = None
        return value

class AttachmentSerializer(ModelSerializer):
    id = serializers.IntegerField(required=False)

    def __init__(self, *args, **kwargs):
        kwargs['allow_null'] = True
        super(AttachmentSerializer, self).__init__(*args, **kwargs)

    class Meta:
        list_serializer_class = AttachmentListSerializer

class TypedAttachmentSerializer(AttachmentSerializer):
    def to_representation(self, obj):
        data = super(TypedAttachmentSerializer, self).to_representation(obj)

        has_parent = (
            self.parent and self.parent.parent and
            hasattr(self.parent.parent.Meta, 'model')
        )
        if has_parent:
            # This is being serialized with its parent object, don't need to
            # reference the parent again.
            pass
        else:
            # Include pointer to parent object. Collapse content_type and
            # object_id into a single attribute: the identifier for the content
            # type plus '_id' e.g. if contenttype's name is 'visit' then the
            # attribute is 'visit_id'.  This lets us pretend the generic
            # foreign key is a regular one in client.  For the value, Use
            # get_object_id insted of obj.object_id, in case the the content
            # object is an IdentifiedModel

            parent_obj = getattr(obj, self.object_field)
            idname = get_ct(parent_obj).identifier
            data[idname + '_id'] = get_object_id(parent_obj)

            # In detail views, include full parent object (without _id suffix)
            if self.is_detail:
                from wq.db import rest
                data[idname] = rest.router.serialize(parent_obj)
        return data

    class Meta:
        # Don't validate these fields (items are saved with their parent)
        exclude = ("content_type", "object_id",)
        list_serializer_class = TypedAttachmentListSerializer

        # patterns-specific meta
        type_field = 'type_id'
        object_field = 'content_object'

class AttachedModelSerializer(ModelSerializer):
    def create(self, validated_data):
        model_data, attachment_data = self.extract_attachments(validated_data)
        instance = super(AttachedModelSerializer, self).create(model_data)

        fields = self.get_fields()
        for name in attachment_data:
            model = fields[name].child.Meta.model
            for attachment in attachment_data[name]:
                if not attachment:
                    continue
                self.set_parent_object(attachment, instance, name)
                self.create_attachment(model, attachment, name)
        return instance

    def update(self, instance, validated_data):
        model_data, attachment_data = self.extract_attachments(validated_data)
        obj = super(
            AttachedModelSerializer, self
        ).update(instance, model_data)

        fields = self.get_fields()
        for name in attachment_data:
            model = fields[name].child.Meta.model
            for attachment in attachment_data[name]:
                if not attachment:
                    continue
                self.set_parent_object(attachment, instance, name)
                if 'id' in attachment:
                    exist = self.get_attachment(model, attachment['id'])
                    self.update_attachment(exist, attachment, name)
                else:
                    self.create_attachment(model, attachment, name)
        return obj

    def extract_attachments(self, validated_data):
        fields = self.get_fields()
        attachment_data = {}
        for name, field in fields.items():
            if isinstance(field, AttachmentListSerializer):
                attachment_data[name] = validated_data.pop(name, [])
        return validated_data, attachment_data

    def set_parent_object(self, attachment, instance, name):
        serializer = self.get_fields()[name].child
        fk_name = getattr(serializer.Meta, 'object_field', 'content_object')
        attachment[fk_name] = instance

    def get_attachment(self, model, pk):
        return model.objects.get(pk=pk)

    def update_attachment(self, exist, attachment, name):
        field = self.get_fields()[name]
        attachment.pop('id')
        field.child.update(exist, attachment)

    def create_attachment(self, model, attachment, name):
        field = self.get_fields()[name]
        field.child.create(attachment)

class NaturalKeyValidator(serializers.UniqueTogetherValidator):
    def set_context(self, serializer):
        self.nested_fields = {
            name: serializer.fields[name]
            for name in self.fields
            if isinstance(serializer.fields[name], NaturalKeySerializer)
        }
        super(NaturalKeyValidator, self).set_context(serializer)

    def filter_queryset(self, attrs, queryset):
        attrs = attrs.copy()
        for field in attrs:
            if field in self.nested_fields:
                assert(isinstance(attrs[field], dict))
                cls = self.nested_fields[field].Meta.model
                result = cls._default_manager.filter(
                    **attrs[field]
                )
                if result.count() == 0:
                    # No existing nested object for these values
                    return queryset.none()
                else:
                    # Existing nested object, use it to validate
                    attrs[field] = result[0].pk

        return super(NaturalKeyValidator, self).filter_queryset(
            attrs, queryset
        )

class NaturalKeySerializer(serializers.ModelSerializer):
    """
    Self-nesting Serializer for NaturalKeyModels
    """
    def build_nested_field(self, field_name, relation_info, nested_depth):
        field_class = NaturalKeySerializer.for_model(
            relation_info.related_model,
            validate_key=False,
        )
        field_kwargs = {}
        return field_class, field_kwargs

    def create(self, validated_data):
        model_class = self.Meta.model
        natural_key_fields = model_class.get_natural_key_fields()
        natural_key = []
        for field in natural_key_fields:
            val = validated_data
            for key in field.split('__'):
                val = val[key]
            natural_key.append(val)
        return model_class.objects.find(*natural_key)

    def update(self, instance, validated_data):
        raise NotImplementedError(
            "Updating an existing natural key is not supported."
        )

    @classmethod
    def for_model(cls, model_class, validate_key=True):
        unique_together = model_class._meta.unique_together[0]

        class Serializer(cls):
            class Meta(cls.Meta):
                model = model_class
                fields = unique_together
                if validate_key:
                    validators = [
                        NaturalKeyValidator(
                            queryset=model_class._default_manager,
                            fields=unique_together,
                        )
                    ]
                else:
                    validators = []
        return Serializer

    @classmethod
    def for_depth(cls, model_class):
        return cls

    def to_internal_value(self, data):
        if html.is_html_input(data):
            data = html.parse_json_form(data)
        result = super(NaturalKeySerializer, self).to_internal_value(data)
        return result

    class Meta:
        depth = 1

class NaturalKeyModelSerializer(AttachedModelSerializer):
    """
    Serializer for models with one or more foreign keys to a NaturalKeyModel
    """
    def build_nested_field(self, field_name, relation_info, nested_depth):
        if issubclass(relation_info.related_model, NaturalKeyModel):
            field_class = NaturalKeySerializer.for_model(
                relation_info.related_model,
                validate_key=False,
            )
            field_kwargs = {}
            if relation_info.model_field.null:
                field_kwargs['required'] = False
            return field_class, field_kwargs

        return super(NaturalKeyModelSerializer, self).build_nested_field(
            field_name, relation_info, nested_depth
        )

    def build_relational_field(self, field_name, relation_info):
        field_class, field_kwargs = super(
            NaturalKeyModelSerializer, self
        ).build_relational_field(
            field_name, relation_info
        )
        if issubclass(relation_info.related_model, NaturalKeyModel):
            field_kwargs.pop('queryset')
            field_kwargs['read_only'] = True
        return field_class, field_kwargs

    def get_fields(self):
        fields = super(NaturalKeyModelSerializer, self).get_fields()
        info = model_meta.get_field_info(self.Meta.model)
        for key in fields:
            if not key.endswith('_id'):
                continue
            field = key[:-3]
            if field in fields or field not in info.relations:
                continue
            relation_info = info.relations[field]
            if not issubclass(relation_info.related_model, NaturalKeyModel):
                continue
            field_class, field_kwargs = self.build_nested_field(
                field, relation_info, 1
            )
            fields[field] = field_class(**field_kwargs)
        return fields

    def create(self, validated_data):
        self.convert_natural_keys(
            validated_data
        )
        return super(NaturalKeyModelSerializer, self).create(
            validated_data
        )

    def update(self, instance, validated_data):
        self.convert_natural_keys(
            validated_data
        )
        return super(NaturalKeyModelSerializer, self).update(
            instance, validated_data
        )

    def convert_natural_keys(self, validated_data):
        fields = self.get_fields()
        for name, field in fields.items():
            if name not in validated_data:
                continue
            if isinstance(field, NaturalKeySerializer):
                validated_data[name] = fields[name].create(
                    validated_data[name]
                )
<commit_msg>fix field lookup<commit_after>from wq.db.rest.serializers import ModelSerializer
from rest_framework import serializers
from rest_framework.utils import model_meta
from wq.db.rest.models import get_ct, get_object_id
from wq.db.rest import compat as html
from .models import NaturalKeyModel

class AttachmentListSerializer(serializers.ListSerializer):
    def to_representation(self, data):
        data = super(AttachmentListSerializer, self).to_representation(data)
        if self.parent:
            for i, row in enumerate(data):
                row['@index'] = i
        return data

class TypedAttachmentListSerializer(AttachmentListSerializer):
    def get_value(self, dictionary):
        # Handle attachments that are submitted together with their parent
        value = super(TypedAttachmentListSerializer, self).get_value(
            dictionary
        )
        if not isinstance(value, list):
            return []
        for i, row in enumerate(value):
            empty = True
            if isinstance(row, dict):
                for key, val in row.items():
                    if key == self.child.Meta.type_field:
                        continue
                    elif val:
                        empty = False
            if empty:
                value[i] = None
        return value

class AttachmentSerializer(ModelSerializer):
    id = serializers.IntegerField(required=False)

    def __init__(self, *args, **kwargs):
        kwargs['allow_null'] = True
        super(AttachmentSerializer, self).__init__(*args, **kwargs)

    class Meta:
        list_serializer_class = AttachmentListSerializer

class TypedAttachmentSerializer(AttachmentSerializer):
    def to_representation(self, obj):
        data = super(TypedAttachmentSerializer, self).to_representation(obj)

        has_parent = (
            self.parent and self.parent.parent and
            hasattr(self.parent.parent.Meta, 'model')
        )
        if has_parent:
            # This is being serialized with its parent object, don't need to
            # reference the parent again.
            pass
        else:
            # Include pointer to parent object. Collapse content_type and
            # object_id into a single attribute: the identifier for the content
            # type plus '_id' e.g. if contenttype's name is 'visit' then the
            # attribute is 'visit_id'.  This lets us pretend the generic
            # foreign key is a regular one in client.  For the value, Use
            # get_object_id insted of obj.object_id, in case the the content
            # object is an IdentifiedModel

            parent_obj = getattr(obj, self.Meta.object_field)
            idname = get_ct(parent_obj).identifier
            data[idname + '_id'] = get_object_id(parent_obj)

            # In detail views, include full parent object (without _id suffix)
            if self.is_detail:
                from wq.db import rest
                data[idname] = rest.router.serialize(parent_obj)
        return data

    class Meta:
        # Don't validate these fields (items are saved with their parent)
        exclude = ("content_type", "object_id",)
        list_serializer_class = TypedAttachmentListSerializer

        # patterns-specific meta
        type_field = 'type_id'
        object_field = 'content_object'

class AttachedModelSerializer(ModelSerializer):
    def create(self, validated_data):
        model_data, attachment_data = self.extract_attachments(validated_data)
        instance = super(AttachedModelSerializer, self).create(model_data)

        fields = self.get_fields()
        for name in attachment_data:
            model = fields[name].child.Meta.model
            for attachment in attachment_data[name]:
                if not attachment:
                    continue
                self.set_parent_object(attachment, instance, name)
                self.create_attachment(model, attachment, name)
        return instance

    def update(self, instance, validated_data):
        model_data, attachment_data = self.extract_attachments(validated_data)
        obj = super(
            AttachedModelSerializer, self
        ).update(instance, model_data)

        fields = self.get_fields()
        for name in attachment_data:
            model = fields[name].child.Meta.model
            for attachment in attachment_data[name]:
                if not attachment:
                    continue
                self.set_parent_object(attachment, instance, name)
                if 'id' in attachment:
                    exist = self.get_attachment(model, attachment['id'])
                    self.update_attachment(exist, attachment, name)
                else:
                    self.create_attachment(model, attachment, name)
        return obj

    def extract_attachments(self, validated_data):
        fields = self.get_fields()
        attachment_data = {}
        for name, field in fields.items():
            if isinstance(field, AttachmentListSerializer):
                attachment_data[name] = validated_data.pop(name, [])
        return validated_data, attachment_data

    def set_parent_object(self, attachment, instance, name):
        serializer = self.get_fields()[name].child
        fk_name = getattr(serializer.Meta, 'object_field', 'content_object')
        attachment[fk_name] = instance

    def get_attachment(self, model, pk):
        return model.objects.get(pk=pk)

    def update_attachment(self, exist, attachment, name):
        field = self.get_fields()[name]
        attachment.pop('id')
        field.child.update(exist, attachment)

    def create_attachment(self, model, attachment, name):
        field = self.get_fields()[name]
        field.child.create(attachment)

class NaturalKeyValidator(serializers.UniqueTogetherValidator):
    def set_context(self, serializer):
        self.nested_fields = {
            name: serializer.fields[name]
            for name in self.fields
            if isinstance(serializer.fields[name], NaturalKeySerializer)
        }
        super(NaturalKeyValidator, self).set_context(serializer)

    def filter_queryset(self, attrs, queryset):
        attrs = attrs.copy()
        for field in attrs:
            if field in self.nested_fields:
                assert(isinstance(attrs[field], dict))
                cls = self.nested_fields[field].Meta.model
                result = cls._default_manager.filter(
                    **attrs[field]
                )
                if result.count() == 0:
                    # No existing nested object for these values
                    return queryset.none()
                else:
                    # Existing nested object, use it to validate
                    attrs[field] = result[0].pk

        return super(NaturalKeyValidator, self).filter_queryset(
            attrs, queryset
        )

class NaturalKeySerializer(serializers.ModelSerializer):
    """
    Self-nesting Serializer for NaturalKeyModels
    """
    def build_nested_field(self, field_name, relation_info, nested_depth):
        field_class = NaturalKeySerializer.for_model(
            relation_info.related_model,
            validate_key=False,
        )
        field_kwargs = {}
        return field_class, field_kwargs

    def create(self, validated_data):
        model_class = self.Meta.model
        natural_key_fields = model_class.get_natural_key_fields()
        natural_key = []
        for field in natural_key_fields:
            val = validated_data
            for key in field.split('__'):
                val = val[key]
            natural_key.append(val)
        return model_class.objects.find(*natural_key)

    def update(self, instance, validated_data):
        raise NotImplementedError(
            "Updating an existing natural key is not supported."
        )

    @classmethod
    def for_model(cls, model_class, validate_key=True):
        unique_together = model_class._meta.unique_together[0]

        class Serializer(cls):
            class Meta(cls.Meta):
                model = model_class
                fields = unique_together
                if validate_key:
                    validators = [
                        NaturalKeyValidator(
                            queryset=model_class._default_manager,
                            fields=unique_together,
                        )
                    ]
                else:
                    validators = []
        return Serializer

    @classmethod
    def for_depth(cls, model_class):
        return cls

    def to_internal_value(self, data):
        if html.is_html_input(data):
            data = html.parse_json_form(data)
        result = super(NaturalKeySerializer, self).to_internal_value(data)
        return result

    class Meta:
        depth = 1

class NaturalKeyModelSerializer(AttachedModelSerializer):
    """
    Serializer for models with one or more foreign keys to a NaturalKeyModel
    """
    def build_nested_field(self, field_name, relation_info, nested_depth):
        if issubclass(relation_info.related_model, NaturalKeyModel):
            field_class = NaturalKeySerializer.for_model(
                relation_info.related_model,
                validate_key=False,
            )
            field_kwargs = {}
            if relation_info.model_field.null:
                field_kwargs['required'] = False
            return field_class, field_kwargs

        return super(NaturalKeyModelSerializer, self).build_nested_field(
            field_name, relation_info, nested_depth
        )

    def build_relational_field(self, field_name, relation_info):
        field_class, field_kwargs = super(
            NaturalKeyModelSerializer, self
        ).build_relational_field(
            field_name, relation_info
        )
        if issubclass(relation_info.related_model, NaturalKeyModel):
            field_kwargs.pop('queryset')
            field_kwargs['read_only'] = True
        return field_class, field_kwargs

    def get_fields(self):
        fields = super(NaturalKeyModelSerializer, self).get_fields()
        info = model_meta.get_field_info(self.Meta.model)
        for key in fields:
            if not key.endswith('_id'):
                continue
            field = key[:-3]
            if field in fields or field not in info.relations:
                continue
            relation_info = info.relations[field]
            if not issubclass(relation_info.related_model, NaturalKeyModel):
                continue
            field_class, field_kwargs = self.build_nested_field(
                field, relation_info, 1
            )
            fields[field] = field_class(**field_kwargs)
        return fields

    def create(self, validated_data):
        self.convert_natural_keys(
            validated_data
        )
        return super(NaturalKeyModelSerializer, self).create(
            validated_data
        )

    def update(self, instance, validated_data):
        self.convert_natural_keys(
            validated_data
        )
        return super(NaturalKeyModelSerializer, self).update(
            instance, validated_data
        )

    def convert_natural_keys(self, validated_data):
        fields = self.get_fields()
        for name, field in fields.items():
            if name not in validated_data:
                continue
            if isinstance(field, NaturalKeySerializer):
                validated_data[name] = fields[name].create(
                    validated_data[name]
                )
<|endoftext|>


<commit_before>from multiprocessing import Lock

import dash
from dash.dependencies import Input, Output
import dash_core_components as dcc
import dash_html_components as html

def test_ldcp001_loading_component_initialization(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [dcc.Loading([html.Div(id="div-1")], className="loading")], id="root"
    )

    @app.callback(Output("div-1", "children"), [Input("root", "n_clicks")])
    def updateDiv(children):
        with lock:
            return "content"

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.find_element(".loading .dash-spinner")
        # ensure inner component is also mounted
        dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.wait_for_text_to_equal("#div-1", "content")

    assert not dash_dcc.get_logs()

def test_ldcp002_loading_component_action(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [dcc.Loading([html.Div(id="div-1")], className="loading")], id="root"
    )

    @app.callback(Output("div-1", "children"), [Input("root", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed"

        return "content"

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.wait_for_text_to_equal("#div-1", "content")

        dash_dcc.find_element("#root").click()

        dash_dcc.find_element(".loading .dash-spinner")
        # mounted but hidden, so looks like no text
        dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.wait_for_text_to_equal("#div-1", "changed")

    assert not dash_dcc.get_logs()

def test_ldcp003_multiple_loading_components(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [
            dcc.Loading([html.Button(id="btn-1")], className="loading-1"),
            dcc.Loading([html.Button(id="btn-2")], className="loading-2"),
        ],
        id="root",
    )

    @app.callback(Output("btn-1", "children"), [Input("btn-2", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 1"

        return "content 1"

    @app.callback(Output("btn-2", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 2"

        return "content 2"

    dash_dcc.start_server(app)

    dash_dcc.wait_for_text_to_equal("#btn-1", "content 1")
    dash_dcc.wait_for_text_to_equal("#btn-2", "content 2")

    with lock:
        dash_dcc.find_element("#btn-1").click()

        dash_dcc.find_element(".loading-2 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-2", "")

    dash_dcc.wait_for_text_to_equal("#btn-2", "changed 2")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-1", "")

    dash_dcc.wait_for_text_to_equal("#btn-1", "changed 1")

    assert not dash_dcc.get_logs()

def test_ldcp004_nested_loading_components(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [
            dcc.Loading(
                [
                    html.Button(id="btn-1"),
                    dcc.Loading([html.Button(id="btn-2")], className="loading-2"),
                ],
                className="loading-1",
            )
        ],
        id="root",
    )

    @app.callback(Output("btn-1", "children"), [Input("btn-2", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 1"

        return "content 1"

    @app.callback(Output("btn-2", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 2"

        return "content 2"

    dash_dcc.start_server(app)

    dash_dcc.wait_for_text_to_equal("#btn-1", "content 1")
    dash_dcc.wait_for_text_to_equal("#btn-2", "content 2")

    with lock:
        dash_dcc.find_element("#btn-1").click()

        dash_dcc.find_element(".loading-2 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-2", "")

    dash_dcc.wait_for_text_to_equal("#btn-2", "changed 2")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-1", "")

    dash_dcc.wait_for_text_to_equal("#btn-1", "changed 1")

    assert not dash_dcc.get_logs()

def test_ldcp005_dynamic_loading_component(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__, suppress_callback_exceptions=True)

    app.layout = html.Div([html.Button(id="btn-1"), html.Div(id="div-1")])

    @app.callback(Output("div-1", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is None:
            return

        with lock:
            return html.Div(
                [
                    html.Button(id="btn-2"),
                    dcc.Loading([html.Button(id="btn-3")], className="loading-1"),
                ]
            )

    @app.callback(Output("btn-3", "children"), [Input("btn-2", "n_clicks")])
    def updateDynamic(n_clicks):
        if n_clicks is None:
            return "content"

        with lock:
            return "changed"

    dash_dcc.start_server(app)

    dash_dcc.find_element("#btn-1")
    dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.find_element("#btn-1").click()

    dash_dcc.find_element("#div-1 #btn-2")
    dash_dcc.wait_for_text_to_equal("#btn-3", "content")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-3", "")

    dash_dcc.wait_for_text_to_equal("#btn-3", "changed")

    assert not dash_dcc.get_logs()

def test_ldcp006_children_identity(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)
    app.layout = html.Div([
        html.Button("click", id="btn"),
        dcc.Loading(dcc.Graph(id="graph"), className="loading")
    ])

    @app.callback(Output("graph", "figure"), [Input("btn", "n_clicks")])
    def update_graph(n):
        with lock:
            bars = list(range(2, (n or 0) + 5))
            return {
                "data": [{"type": "bar", "x": bars, "y": bars}],
                "layout": {"width": 400, "height": 400}
            }

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.find_element(".loading .dash-spinner")
        dash_dcc.find_element("#graph .js-plotly-plot")
        dash_dcc.driver.execute_script(
            "window.gd = document.querySelector('.js-plotly-plot');"
            "window.gd.__test__ = 'boo';"
        )

    test_identity = (
        "var gd_ = document.querySelector('.js-plotly-plot');"
        "return gd_ === window.gd && gd_.__test__ === 'boo';"
    )

    assert len(dash_dcc.find_elements('.js-plotly-plot .bars path')) == 3
    assert dash_dcc.driver.execute_script(test_identity)

    with lock:
        dash_dcc.find_element("#btn").click()
        dash_dcc.find_element(".loading .dash-spinner")
        assert len(dash_dcc.find_elements('.js-plotly-plot .bars path')) == 3
        assert dash_dcc.driver.execute_script(test_identity)

    assert len(dash_dcc.find_elements('.js-plotly-plot .bars path')) == 4
    assert dash_dcc.driver.execute_script(test_identity)
<commit_msg>include computed visibility in the loading/graph test<commit_after>from multiprocessing import Lock

import dash
from dash.dependencies import Input, Output
import dash_core_components as dcc
import dash_html_components as html

def test_ldcp001_loading_component_initialization(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [dcc.Loading([html.Div(id="div-1")], className="loading")], id="root"
    )

    @app.callback(Output("div-1", "children"), [Input("root", "n_clicks")])
    def updateDiv(children):
        with lock:
            return "content"

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.find_element(".loading .dash-spinner")
        # ensure inner component is also mounted
        dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.wait_for_text_to_equal("#div-1", "content")

    assert not dash_dcc.get_logs()

def test_ldcp002_loading_component_action(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [dcc.Loading([html.Div(id="div-1")], className="loading")], id="root"
    )

    @app.callback(Output("div-1", "children"), [Input("root", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed"

        return "content"

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.wait_for_text_to_equal("#div-1", "content")

        dash_dcc.find_element("#root").click()

        dash_dcc.find_element(".loading .dash-spinner")
        # mounted but hidden, so looks like no text
        dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.wait_for_text_to_equal("#div-1", "changed")

    assert not dash_dcc.get_logs()

def test_ldcp003_multiple_loading_components(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [
            dcc.Loading([html.Button(id="btn-1")], className="loading-1"),
            dcc.Loading([html.Button(id="btn-2")], className="loading-2"),
        ],
        id="root",
    )

    @app.callback(Output("btn-1", "children"), [Input("btn-2", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 1"

        return "content 1"

    @app.callback(Output("btn-2", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 2"

        return "content 2"

    dash_dcc.start_server(app)

    dash_dcc.wait_for_text_to_equal("#btn-1", "content 1")
    dash_dcc.wait_for_text_to_equal("#btn-2", "content 2")

    with lock:
        dash_dcc.find_element("#btn-1").click()

        dash_dcc.find_element(".loading-2 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-2", "")

    dash_dcc.wait_for_text_to_equal("#btn-2", "changed 2")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-1", "")

    dash_dcc.wait_for_text_to_equal("#btn-1", "changed 1")

    assert not dash_dcc.get_logs()

def test_ldcp004_nested_loading_components(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)

    app.layout = html.Div(
        [
            dcc.Loading(
                [
                    html.Button(id="btn-1"),
                    dcc.Loading([html.Button(id="btn-2")], className="loading-2"),
                ],
                className="loading-1",
            )
        ],
        id="root",
    )

    @app.callback(Output("btn-1", "children"), [Input("btn-2", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 1"

        return "content 1"

    @app.callback(Output("btn-2", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is not None:
            with lock:
                return "changed 2"

        return "content 2"

    dash_dcc.start_server(app)

    dash_dcc.wait_for_text_to_equal("#btn-1", "content 1")
    dash_dcc.wait_for_text_to_equal("#btn-2", "content 2")

    with lock:
        dash_dcc.find_element("#btn-1").click()

        dash_dcc.find_element(".loading-2 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-2", "")

    dash_dcc.wait_for_text_to_equal("#btn-2", "changed 2")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-1", "")

    dash_dcc.wait_for_text_to_equal("#btn-1", "changed 1")

    assert not dash_dcc.get_logs()

def test_ldcp005_dynamic_loading_component(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__, suppress_callback_exceptions=True)

    app.layout = html.Div([html.Button(id="btn-1"), html.Div(id="div-1")])

    @app.callback(Output("div-1", "children"), [Input("btn-1", "n_clicks")])
    def updateDiv(n_clicks):
        if n_clicks is None:
            return

        with lock:
            return html.Div(
                [
                    html.Button(id="btn-2"),
                    dcc.Loading([html.Button(id="btn-3")], className="loading-1"),
                ]
            )

    @app.callback(Output("btn-3", "children"), [Input("btn-2", "n_clicks")])
    def updateDynamic(n_clicks):
        if n_clicks is None:
            return "content"

        with lock:
            return "changed"

    dash_dcc.start_server(app)

    dash_dcc.find_element("#btn-1")
    dash_dcc.wait_for_text_to_equal("#div-1", "")

    dash_dcc.find_element("#btn-1").click()

    dash_dcc.find_element("#div-1 #btn-2")
    dash_dcc.wait_for_text_to_equal("#btn-3", "content")

    with lock:
        dash_dcc.find_element("#btn-2").click()

        dash_dcc.find_element(".loading-1 .dash-spinner")
        dash_dcc.wait_for_text_to_equal("#btn-3", "")

    dash_dcc.wait_for_text_to_equal("#btn-3", "changed")

    assert not dash_dcc.get_logs()

def test_ldcp006_children_identity(dash_dcc):
    lock = Lock()

    app = dash.Dash(__name__)
    app.layout = html.Div(
        [
            html.Button("click", id="btn"),
            dcc.Loading(dcc.Graph(id="graph"), className="loading"),
        ]
    )

    @app.callback(Output("graph", "figure"), [Input("btn", "n_clicks")])
    def update_graph(n):
        with lock:
            bars = list(range(2, (n or 0) + 5))
            return {
                "data": [{"type": "bar", "x": bars, "y": bars}],
                "layout": {"width": 400, "height": 400},
            }

    def get_graph_visibility():
        return dash_dcc.driver.execute_script(
            "var gd_ = document.querySelector('.js-plotly-plot');"
            "return getComputedStyle(gd_).visibility;"
        )

    with lock:
        dash_dcc.start_server(app)
        dash_dcc.find_element(".loading .dash-spinner")
        dash_dcc.find_element("#graph .js-plotly-plot")
        dash_dcc.driver.execute_script(
            "window.gd = document.querySelector('.js-plotly-plot');"
            "window.gd.__test__ = 'boo';"
        )
        assert get_graph_visibility() == "hidden"

    test_identity = (
        "var gd_ = document.querySelector('.js-plotly-plot');"
        "return gd_ === window.gd && gd_.__test__ === 'boo';"
    )

    assert len(dash_dcc.find_elements(".js-plotly-plot .bars path")) == 3
    assert dash_dcc.driver.execute_script(test_identity)
    assert get_graph_visibility() == "visible"

    with lock:
        dash_dcc.find_element("#btn").click()
        dash_dcc.find_element(".loading .dash-spinner")
        assert len(dash_dcc.find_elements(".js-plotly-plot .bars path")) == 3
        assert dash_dcc.driver.execute_script(test_identity)
        assert get_graph_visibility() == "hidden"

    assert len(dash_dcc.find_elements(".js-plotly-plot .bars path")) == 4
    assert dash_dcc.driver.execute_script(test_identity)
    assert get_graph_visibility() == "visible"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# winapi.py: Windows API-Python interface (removes dependency on pywin32)
#
# Copyright (C) 2007 Thomas Heller <theller@ctypes.org>
# Copyright (C) 2010 Will McGugan <will@willmcgugan.com>
# Copyright (C) 2010 Ryan Kelly <ryan@rfk.id.au>
# Copyright (C) 2010 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright (C) 2014 Thomas Amland
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and / or other materials provided with the distribution.
# * Neither the name of the organization nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Portions of this code were taken from pyfilesystem, which uses the above
# new BSD license.

from __future__ import with_statement

import ctypes.wintypes
import struct

try:
    LPVOID = ctypes.wintypes.LPVOID
except AttributeError:
    # LPVOID wasn't defined in Py2.5, guess it was introduced in Py2.6
    LPVOID = ctypes.c_void_p

# Invalid handle value.
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value

# File notification contants.
FILE_NOTIFY_CHANGE_FILE_NAME = 0x01
FILE_NOTIFY_CHANGE_DIR_NAME = 0x02
FILE_NOTIFY_CHANGE_ATTRIBUTES = 0x04
FILE_NOTIFY_CHANGE_SIZE = 0x08
FILE_NOTIFY_CHANGE_LAST_WRITE = 0x010
FILE_NOTIFY_CHANGE_LAST_ACCESS = 0x020
FILE_NOTIFY_CHANGE_CREATION = 0x040
FILE_NOTIFY_CHANGE_SECURITY = 0x0100

FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
FILE_FLAG_OVERLAPPED = 0x40000000
FILE_LIST_DIRECTORY = 0x01
FILE_SHARE_READ = 0x01
FILE_SHARE_WRITE = 0x02
FILE_SHARE_DELETE = 0x04
OPEN_EXISTING = 3

# File action constants.
FILE_ACTION_CREATED = 1
FILE_ACTION_DELETED = 2
FILE_ACTION_MODIFIED = 3
FILE_ACTION_RENAMED_OLD_NAME = 4
FILE_ACTION_RENAMED_NEW_NAME = 5
FILE_ACTION_OVERFLOW = 0xFFFF

# Aliases
FILE_ACTION_ADDED = FILE_ACTION_CREATED
FILE_ACTION_REMOVED = FILE_ACTION_DELETED

THREAD_TERMINATE = 0x0001

# IO waiting constants.
WAIT_ABANDONED = 0x00000080
WAIT_IO_COMPLETION = 0x000000C0
WAIT_OBJECT_0 = 0x00000000
WAIT_TIMEOUT = 0x00000102

# Error codes
ERROR_OPERATION_ABORTED = 995

class OVERLAPPED(ctypes.Structure):
    _fields_ = [('Internal', LPVOID),
                ('InternalHigh', LPVOID),
                ('Offset', ctypes.wintypes.DWORD),
                ('OffsetHigh', ctypes.wintypes.DWORD),
                ('Pointer', LPVOID),
                ('hEvent', ctypes.wintypes.HANDLE),
                ]

def _errcheck_bool(value, func, args):
    if not value:
        raise ctypes.WinError()
    return args

def _errcheck_handle(value, func, args):
    if not value:
        raise ctypes.WinError()
    if value == INVALID_HANDLE_VALUE:
        raise ctypes.WinError()
    return args

def _errcheck_dword(value, func, args):
    if value == 0xFFFFFFFF:
        raise ctypes.WinError()
    return args

try:
    ReadDirectoryChangesW = ctypes.windll.kernel32.ReadDirectoryChangesW
except AttributeError:
    raise ImportError("ReadDirectoryChangesW is not available")

ReadDirectoryChangesW.restype = ctypes.wintypes.BOOL
ReadDirectoryChangesW.errcheck = _errcheck_bool
ReadDirectoryChangesW.argtypes = (
    ctypes.wintypes.HANDLE,  # hDirectory
    LPVOID,  # lpBuffer
    ctypes.wintypes.DWORD,  # nBufferLength
    ctypes.wintypes.BOOL,  # bWatchSubtree
    ctypes.wintypes.DWORD,  # dwNotifyFilter
    ctypes.POINTER(ctypes.wintypes.DWORD),  # lpBytesReturned
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
    LPVOID  # FileIOCompletionRoutine # lpCompletionRoutine
)

CreateFileW = ctypes.windll.kernel32.CreateFileW
CreateFileW.restype = ctypes.wintypes.HANDLE
CreateFileW.errcheck = _errcheck_handle
CreateFileW.argtypes = (
    ctypes.wintypes.LPCWSTR,  # lpFileName
    ctypes.wintypes.DWORD,  # dwDesiredAccess
    ctypes.wintypes.DWORD,  # dwShareMode
    LPVOID,  # lpSecurityAttributes
    ctypes.wintypes.DWORD,  # dwCreationDisposition
    ctypes.wintypes.DWORD,  # dwFlagsAndAttributes
    ctypes.wintypes.HANDLE  # hTemplateFile
)

CloseHandle = ctypes.windll.kernel32.CloseHandle
CloseHandle.restype = ctypes.wintypes.BOOL
CloseHandle.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
)

CancelIoEx = ctypes.windll.kernel32.CancelIoEx
CancelIoEx.restype = ctypes.wintypes.BOOL
CancelIoEx.errcheck = _errcheck_bool
CancelIoEx.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
    ctypes.POINTER(OVERLAPPED)  # lpOverlapped
)

CreateEvent = ctypes.windll.kernel32.CreateEventW
CreateEvent.restype = ctypes.wintypes.HANDLE
CreateEvent.errcheck = _errcheck_handle
CreateEvent.argtypes = (
    LPVOID,  # lpEventAttributes
    ctypes.wintypes.BOOL,  # bManualReset
    ctypes.wintypes.BOOL,  # bInitialState
    ctypes.wintypes.LPCWSTR,  # lpName
)

SetEvent = ctypes.windll.kernel32.SetEvent
SetEvent.restype = ctypes.wintypes.BOOL
SetEvent.errcheck = _errcheck_bool
SetEvent.argtypes = (
    ctypes.wintypes.HANDLE,  # hEvent
)

WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
WaitForSingleObjectEx.restype = ctypes.wintypes.DWORD
WaitForSingleObjectEx.errcheck = _errcheck_dword
WaitForSingleObjectEx.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
    ctypes.wintypes.DWORD,  # dwMilliseconds
    ctypes.wintypes.BOOL,  # bAlertable
)

CreateIoCompletionPort = ctypes.windll.kernel32.CreateIoCompletionPort
CreateIoCompletionPort.restype = ctypes.wintypes.HANDLE
CreateIoCompletionPort.errcheck = _errcheck_handle
CreateIoCompletionPort.argtypes = (
    ctypes.wintypes.HANDLE,  # FileHandle
    ctypes.wintypes.HANDLE,  # ExistingCompletionPort
    LPVOID,  # CompletionKey
    ctypes.wintypes.DWORD,  # NumberOfConcurrentThreads
)

GetQueuedCompletionStatus = ctypes.windll.kernel32.GetQueuedCompletionStatus
GetQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
GetQueuedCompletionStatus.errcheck = _errcheck_bool
GetQueuedCompletionStatus.argtypes = (
    ctypes.wintypes.HANDLE,  # CompletionPort
    LPVOID,  # lpNumberOfBytesTransferred
    LPVOID,  # lpCompletionKey
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
    ctypes.wintypes.DWORD,  # dwMilliseconds
)

PostQueuedCompletionStatus = ctypes.windll.kernel32.PostQueuedCompletionStatus
PostQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
PostQueuedCompletionStatus.errcheck = _errcheck_bool
PostQueuedCompletionStatus.argtypes = (
    ctypes.wintypes.HANDLE,  # CompletionPort
    ctypes.wintypes.DWORD,  # lpNumberOfBytesTransferred
    ctypes.wintypes.DWORD,  # lpCompletionKey
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
)

class FILE_NOTIFY_INFORMATION(ctypes.Structure):
    _fields_ = [("NextEntryOffset", ctypes.wintypes.DWORD),
                ("Action", ctypes.wintypes.DWORD),
                ("FileNameLength", ctypes.wintypes.DWORD),
                #("FileName", (ctypes.wintypes.WCHAR * 1))]
                ("FileName", (ctypes.c_char * 1))]

LPFNI = ctypes.POINTER(FILE_NOTIFY_INFORMATION)

# We don't need to recalculate these flags every time a call is made to
# the win32 API functions.
WATCHDOG_FILE_FLAGS = FILE_FLAG_BACKUP_SEMANTICS
WATCHDOG_FILE_SHARE_FLAGS = reduce(
    lambda x, y: x | y, [
        FILE_SHARE_READ,
        FILE_SHARE_WRITE,
        FILE_SHARE_DELETE,
    ])
WATCHDOG_FILE_NOTIFY_FLAGS = reduce(
    lambda x, y: x | y, [
        FILE_NOTIFY_CHANGE_FILE_NAME,
        FILE_NOTIFY_CHANGE_DIR_NAME,
        FILE_NOTIFY_CHANGE_ATTRIBUTES,
        FILE_NOTIFY_CHANGE_SIZE,
        FILE_NOTIFY_CHANGE_LAST_WRITE,
        FILE_NOTIFY_CHANGE_SECURITY,
        FILE_NOTIFY_CHANGE_LAST_ACCESS,
        FILE_NOTIFY_CHANGE_CREATION,
    ])

BUFFER_SIZE = 2048

def _parse_event_buffer(readBuffer, nBytes):
    results = []
    while nBytes > 0:
        fni = ctypes.cast(readBuffer, LPFNI)[0]
        ptr = ctypes.addressof(fni) + FILE_NOTIFY_INFORMATION.FileName.offset
        #filename = ctypes.wstring_at(ptr, fni.FileNameLength)
        filename = ctypes.string_at(ptr, fni.FileNameLength)
        results.append((fni.Action, filename.decode('utf-16')))
        numToSkip = fni.NextEntryOffset
        if numToSkip <= 0:
            break
        readBuffer = readBuffer[numToSkip:]
        nBytes -= numToSkip  # numToSkip is long. nBytes should be long too.
    return results

def get_directory_handle(path):
    """Returns a Windows handle to the specified directory path."""
    return CreateFileW(path, FILE_LIST_DIRECTORY, WATCHDOG_FILE_SHARE_FLAGS,
                       None, OPEN_EXISTING, WATCHDOG_FILE_FLAGS, None)

def close_directory_handle(handle):
    try:
        CancelIoEx(handle, None)  # force ReadDirectoryChangesW to return
    except WindowsError:
        return

def read_directory_changes(handle, recursive):
    """Read changes to the directory using the specified directory handle.

    http://timgolden.me.uk/pywin32-docs/win32file__ReadDirectoryChangesW_meth.html
    """
    event_buffer = ctypes.create_string_buffer(BUFFER_SIZE)
    nbytes = ctypes.wintypes.DWORD()
    try:
        ReadDirectoryChangesW(handle, ctypes.byref(event_buffer),
                              len(event_buffer), recursive,
                              WATCHDOG_FILE_NOTIFY_FLAGS,
                              ctypes.byref(nbytes), None, None)
    except WindowsError as e:
        if e.winerror == ERROR_OPERATION_ABORTED:
            return [], 0
        raise e

    # Python 2/3 compat
    try:
        int_class = long
    except NameError:
        int_class = int
    return event_buffer.raw, int_class(nbytes.value)

class WinAPINativeEvent(object):
    def __init__(self, action, src_path):
        self.action = action
        self.src_path = src_path

    @property
    def is_added(self):
        return self.action == FILE_ACTION_CREATED

    @property
    def is_removed(self):
        return self.action == FILE_ACTION_REMOVED

    @property
    def is_modified(self):
        return self.action == FILE_ACTION_MODIFIED

    @property
    def is_renamed_old(self):
        return self.action == FILE_ACTION_RENAMED_OLD_NAME

    @property
    def is_renamed_new(self):
        return self.action == FILE_ACTION_RENAMED_NEW_NAME

    def __repr__(self):
        return ("<WinAPINativeEvent: action=%d, src_path=%s>" % (self.action, self.src_path))

def read_events(handle, recursive):
    buf, nbytes = read_directory_changes(handle, recursive)
    events = _parse_event_buffer(buf, nbytes)
    return [WinAPINativeEvent(action, path) for action, path in events]
<commit_msg>[win] use repr to make event printable<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# winapi.py: Windows API-Python interface (removes dependency on pywin32)
#
# Copyright (C) 2007 Thomas Heller <theller@ctypes.org>
# Copyright (C) 2010 Will McGugan <will@willmcgugan.com>
# Copyright (C) 2010 Ryan Kelly <ryan@rfk.id.au>
# Copyright (C) 2010 Yesudeep Mangalapilly <yesudeep@gmail.com>
# Copyright (C) 2014 Thomas Amland
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and / or other materials provided with the distribution.
# * Neither the name of the organization nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Portions of this code were taken from pyfilesystem, which uses the above
# new BSD license.

from __future__ import with_statement

import ctypes.wintypes
import struct

try:
    LPVOID = ctypes.wintypes.LPVOID
except AttributeError:
    # LPVOID wasn't defined in Py2.5, guess it was introduced in Py2.6
    LPVOID = ctypes.c_void_p

# Invalid handle value.
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value

# File notification contants.
FILE_NOTIFY_CHANGE_FILE_NAME = 0x01
FILE_NOTIFY_CHANGE_DIR_NAME = 0x02
FILE_NOTIFY_CHANGE_ATTRIBUTES = 0x04
FILE_NOTIFY_CHANGE_SIZE = 0x08
FILE_NOTIFY_CHANGE_LAST_WRITE = 0x010
FILE_NOTIFY_CHANGE_LAST_ACCESS = 0x020
FILE_NOTIFY_CHANGE_CREATION = 0x040
FILE_NOTIFY_CHANGE_SECURITY = 0x0100

FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
FILE_FLAG_OVERLAPPED = 0x40000000
FILE_LIST_DIRECTORY = 0x01
FILE_SHARE_READ = 0x01
FILE_SHARE_WRITE = 0x02
FILE_SHARE_DELETE = 0x04
OPEN_EXISTING = 3

# File action constants.
FILE_ACTION_CREATED = 1
FILE_ACTION_DELETED = 2
FILE_ACTION_MODIFIED = 3
FILE_ACTION_RENAMED_OLD_NAME = 4
FILE_ACTION_RENAMED_NEW_NAME = 5
FILE_ACTION_OVERFLOW = 0xFFFF

# Aliases
FILE_ACTION_ADDED = FILE_ACTION_CREATED
FILE_ACTION_REMOVED = FILE_ACTION_DELETED

THREAD_TERMINATE = 0x0001

# IO waiting constants.
WAIT_ABANDONED = 0x00000080
WAIT_IO_COMPLETION = 0x000000C0
WAIT_OBJECT_0 = 0x00000000
WAIT_TIMEOUT = 0x00000102

# Error codes
ERROR_OPERATION_ABORTED = 995

class OVERLAPPED(ctypes.Structure):
    _fields_ = [('Internal', LPVOID),
                ('InternalHigh', LPVOID),
                ('Offset', ctypes.wintypes.DWORD),
                ('OffsetHigh', ctypes.wintypes.DWORD),
                ('Pointer', LPVOID),
                ('hEvent', ctypes.wintypes.HANDLE),
                ]

def _errcheck_bool(value, func, args):
    if not value:
        raise ctypes.WinError()
    return args

def _errcheck_handle(value, func, args):
    if not value:
        raise ctypes.WinError()
    if value == INVALID_HANDLE_VALUE:
        raise ctypes.WinError()
    return args

def _errcheck_dword(value, func, args):
    if value == 0xFFFFFFFF:
        raise ctypes.WinError()
    return args

try:
    ReadDirectoryChangesW = ctypes.windll.kernel32.ReadDirectoryChangesW
except AttributeError:
    raise ImportError("ReadDirectoryChangesW is not available")

ReadDirectoryChangesW.restype = ctypes.wintypes.BOOL
ReadDirectoryChangesW.errcheck = _errcheck_bool
ReadDirectoryChangesW.argtypes = (
    ctypes.wintypes.HANDLE,  # hDirectory
    LPVOID,  # lpBuffer
    ctypes.wintypes.DWORD,  # nBufferLength
    ctypes.wintypes.BOOL,  # bWatchSubtree
    ctypes.wintypes.DWORD,  # dwNotifyFilter
    ctypes.POINTER(ctypes.wintypes.DWORD),  # lpBytesReturned
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
    LPVOID  # FileIOCompletionRoutine # lpCompletionRoutine
)

CreateFileW = ctypes.windll.kernel32.CreateFileW
CreateFileW.restype = ctypes.wintypes.HANDLE
CreateFileW.errcheck = _errcheck_handle
CreateFileW.argtypes = (
    ctypes.wintypes.LPCWSTR,  # lpFileName
    ctypes.wintypes.DWORD,  # dwDesiredAccess
    ctypes.wintypes.DWORD,  # dwShareMode
    LPVOID,  # lpSecurityAttributes
    ctypes.wintypes.DWORD,  # dwCreationDisposition
    ctypes.wintypes.DWORD,  # dwFlagsAndAttributes
    ctypes.wintypes.HANDLE  # hTemplateFile
)

CloseHandle = ctypes.windll.kernel32.CloseHandle
CloseHandle.restype = ctypes.wintypes.BOOL
CloseHandle.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
)

CancelIoEx = ctypes.windll.kernel32.CancelIoEx
CancelIoEx.restype = ctypes.wintypes.BOOL
CancelIoEx.errcheck = _errcheck_bool
CancelIoEx.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
    ctypes.POINTER(OVERLAPPED)  # lpOverlapped
)

CreateEvent = ctypes.windll.kernel32.CreateEventW
CreateEvent.restype = ctypes.wintypes.HANDLE
CreateEvent.errcheck = _errcheck_handle
CreateEvent.argtypes = (
    LPVOID,  # lpEventAttributes
    ctypes.wintypes.BOOL,  # bManualReset
    ctypes.wintypes.BOOL,  # bInitialState
    ctypes.wintypes.LPCWSTR,  # lpName
)

SetEvent = ctypes.windll.kernel32.SetEvent
SetEvent.restype = ctypes.wintypes.BOOL
SetEvent.errcheck = _errcheck_bool
SetEvent.argtypes = (
    ctypes.wintypes.HANDLE,  # hEvent
)

WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
WaitForSingleObjectEx.restype = ctypes.wintypes.DWORD
WaitForSingleObjectEx.errcheck = _errcheck_dword
WaitForSingleObjectEx.argtypes = (
    ctypes.wintypes.HANDLE,  # hObject
    ctypes.wintypes.DWORD,  # dwMilliseconds
    ctypes.wintypes.BOOL,  # bAlertable
)

CreateIoCompletionPort = ctypes.windll.kernel32.CreateIoCompletionPort
CreateIoCompletionPort.restype = ctypes.wintypes.HANDLE
CreateIoCompletionPort.errcheck = _errcheck_handle
CreateIoCompletionPort.argtypes = (
    ctypes.wintypes.HANDLE,  # FileHandle
    ctypes.wintypes.HANDLE,  # ExistingCompletionPort
    LPVOID,  # CompletionKey
    ctypes.wintypes.DWORD,  # NumberOfConcurrentThreads
)

GetQueuedCompletionStatus = ctypes.windll.kernel32.GetQueuedCompletionStatus
GetQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
GetQueuedCompletionStatus.errcheck = _errcheck_bool
GetQueuedCompletionStatus.argtypes = (
    ctypes.wintypes.HANDLE,  # CompletionPort
    LPVOID,  # lpNumberOfBytesTransferred
    LPVOID,  # lpCompletionKey
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
    ctypes.wintypes.DWORD,  # dwMilliseconds
)

PostQueuedCompletionStatus = ctypes.windll.kernel32.PostQueuedCompletionStatus
PostQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
PostQueuedCompletionStatus.errcheck = _errcheck_bool
PostQueuedCompletionStatus.argtypes = (
    ctypes.wintypes.HANDLE,  # CompletionPort
    ctypes.wintypes.DWORD,  # lpNumberOfBytesTransferred
    ctypes.wintypes.DWORD,  # lpCompletionKey
    ctypes.POINTER(OVERLAPPED),  # lpOverlapped
)

class FILE_NOTIFY_INFORMATION(ctypes.Structure):
    _fields_ = [("NextEntryOffset", ctypes.wintypes.DWORD),
                ("Action", ctypes.wintypes.DWORD),
                ("FileNameLength", ctypes.wintypes.DWORD),
                #("FileName", (ctypes.wintypes.WCHAR * 1))]
                ("FileName", (ctypes.c_char * 1))]

LPFNI = ctypes.POINTER(FILE_NOTIFY_INFORMATION)

# We don't need to recalculate these flags every time a call is made to
# the win32 API functions.
WATCHDOG_FILE_FLAGS = FILE_FLAG_BACKUP_SEMANTICS
WATCHDOG_FILE_SHARE_FLAGS = reduce(
    lambda x, y: x | y, [
        FILE_SHARE_READ,
        FILE_SHARE_WRITE,
        FILE_SHARE_DELETE,
    ])
WATCHDOG_FILE_NOTIFY_FLAGS = reduce(
    lambda x, y: x | y, [
        FILE_NOTIFY_CHANGE_FILE_NAME,
        FILE_NOTIFY_CHANGE_DIR_NAME,
        FILE_NOTIFY_CHANGE_ATTRIBUTES,
        FILE_NOTIFY_CHANGE_SIZE,
        FILE_NOTIFY_CHANGE_LAST_WRITE,
        FILE_NOTIFY_CHANGE_SECURITY,
        FILE_NOTIFY_CHANGE_LAST_ACCESS,
        FILE_NOTIFY_CHANGE_CREATION,
    ])

BUFFER_SIZE = 2048

def _parse_event_buffer(readBuffer, nBytes):
    results = []
    while nBytes > 0:
        fni = ctypes.cast(readBuffer, LPFNI)[0]
        ptr = ctypes.addressof(fni) + FILE_NOTIFY_INFORMATION.FileName.offset
        #filename = ctypes.wstring_at(ptr, fni.FileNameLength)
        filename = ctypes.string_at(ptr, fni.FileNameLength)
        results.append((fni.Action, filename.decode('utf-16')))
        numToSkip = fni.NextEntryOffset
        if numToSkip <= 0:
            break
        readBuffer = readBuffer[numToSkip:]
        nBytes -= numToSkip  # numToSkip is long. nBytes should be long too.
    return results

def get_directory_handle(path):
    """Returns a Windows handle to the specified directory path."""
    return CreateFileW(path, FILE_LIST_DIRECTORY, WATCHDOG_FILE_SHARE_FLAGS,
                       None, OPEN_EXISTING, WATCHDOG_FILE_FLAGS, None)

def close_directory_handle(handle):
    try:
        CancelIoEx(handle, None)  # force ReadDirectoryChangesW to return
    except WindowsError:
        return

def read_directory_changes(handle, recursive):
    """Read changes to the directory using the specified directory handle.

    http://timgolden.me.uk/pywin32-docs/win32file__ReadDirectoryChangesW_meth.html
    """
    event_buffer = ctypes.create_string_buffer(BUFFER_SIZE)
    nbytes = ctypes.wintypes.DWORD()
    try:
        ReadDirectoryChangesW(handle, ctypes.byref(event_buffer),
                              len(event_buffer), recursive,
                              WATCHDOG_FILE_NOTIFY_FLAGS,
                              ctypes.byref(nbytes), None, None)
    except WindowsError as e:
        if e.winerror == ERROR_OPERATION_ABORTED:
            return [], 0
        raise e

    # Python 2/3 compat
    try:
        int_class = long
    except NameError:
        int_class = int
    return event_buffer.raw, int_class(nbytes.value)

class WinAPINativeEvent(object):
    def __init__(self, action, src_path):
        self.action = action
        self.src_path = src_path

    @property
    def is_added(self):
        return self.action == FILE_ACTION_CREATED

    @property
    def is_removed(self):
        return self.action == FILE_ACTION_REMOVED

    @property
    def is_modified(self):
        return self.action == FILE_ACTION_MODIFIED

    @property
    def is_renamed_old(self):
        return self.action == FILE_ACTION_RENAMED_OLD_NAME

    @property
    def is_renamed_new(self):
        return self.action == FILE_ACTION_RENAMED_NEW_NAME

    def __repr__(self):
        return ("<WinAPINativeEvent: action=%d, src_path=%r>" % (self.action, self.src_path))

def read_events(handle, recursive):
    buf, nbytes = read_directory_changes(handle, recursive)
    events = _parse_event_buffer(buf, nbytes)
    return [WinAPINativeEvent(action, path) for action, path in events]
<|endoftext|>