<commit_before>#!/usr/bin/python
#
# Copyright 2016 Google Inc.  All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a build from the given commands.

A command is either an addition or a subtraction.  An addition is prefixed with
a +; a subtraction is when prefixed with a -.  After the character, there is a
name of a file or a @ sign and the name of a build file.

Build files are the files found in build/types.  These files are simply a
newline separated list of commands to execute.  So if the "+@complete" command
is given, it will open the complete file and run it (which may in turn open
other build files).  Subtracting a build file will reverse all actions applied
by the given file.  So "-@networking" will remove all the networking plugins.

The core library is always included so does not have to be listed.  The default
is to use the name 'compiled'; if no commands are given, it will build the
complete build.

Examples:
  # Equivalent to +@complete
  build.py

  build.py +@complete
  build.py +@complete -@networking
  build.py --name custom +@manifests +@networking +../my_plugin.js
"""

import os
import re
import shutil
import subprocess
import sys

import shakaBuildHelpers

common_closure_opts = [
    '--language_in', 'ECMASCRIPT5',
    '--language_out', 'ECMASCRIPT3',

    '--jscomp_error=*',

    # 'deprecatedAnnotations' controls complains about @expose, but the new
    # @nocollapse annotation does not do the same job for properties.
    # So since we can't use the new annotations, we have to ignore complaints
    # about the old one.
    '--jscomp_off=deprecatedAnnotations',

    # Analyzer checks require explicit nullability, which is a pain.
    '--jscomp_off=analyzerChecksInternal',

    '--extra_annotation_name=listens',
    '--extra_annotation_name=exportDoc',

    '--conformance_configs',
    ('%s/build/conformance.textproto' %
     shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())),

    '--generate_exports',

    '-D', 'COMPILED=true',
    '-D', 'goog.STRICT_MODE_COMPATIBLE=true',
    '-D', 'goog.ENABLE_DEBUG_LOADER=false',
    '-D', 'GIT_VERSION="%s"' % shakaBuildHelpers.calculate_version()
]
debug_closure_opts = [
    # Don't use a wrapper script in debug mode so all the internals are visible
    # on the global object.

    '-O', 'SIMPLE',
    '-D', 'goog.DEBUG=true',
    '-D', 'goog.asserts.ENABLE_ASSERTS=true',
    '-D', 'shaka.log.MAX_LOG_LEVEL=4',  # shaka.log.Level.DEBUG
]
release_closure_opts = [
    ('--output_wrapper_file=%s/build/wrapper.template.js' %
     shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())),

    '-O', 'ADVANCED',
    '-D', 'goog.DEBUG=false',
    '-D', 'goog.asserts.ENABLE_ASSERTS=false',
    '-D', 'shaka.log.MAX_LOG_LEVEL=0',
]

class Build(object):
  """Defines a build that has been parsed from a build file.

  This has exclude files even though it will not be used at the top-level.  This
  allows combining builds.  A file will only exist in at most one set.

  Members:
    include - A set of files to include.
    exclude - A set of files to remove.
  """

  def __init__(self, include=None, exclude=None):
    self.include = include or set()
    self.exclude = exclude or set()

  def _get_build_file_path(self, name, root):
    """Gets the full path to a build file, if it exists.

    Args:
      name: The string name to check.
      root: The full path to the base directory.

    Returns:
      The full path to the build file, or None if not found.
    """
    source_base = shakaBuildHelpers.get_source_base()
    local_path = os.path.join(root, name)
    build_path = os.path.join(source_base, 'build', 'types', name)
    if (os.path.isfile(local_path) and os.path.isfile(build_path)
        and local_path != build_path):
      print >> sys.stderr, 'Build file "%s" is ambiguous' % name
      return None
    elif os.path.isfile(local_path):
      return local_path
    elif os.path.isfile(build_path):
      return build_path
    else:
      print >> sys.stderr, 'Build file not found: ' + name
      return None

  def _combine(self, other):
    include_all = self.include | other.include
    exclude_all = self.exclude | other.exclude
    self.include = include_all - exclude_all
    self.exclude = exclude_all - include_all

  def reverse(self):
    return Build(self.exclude, self.include)

  def add_core(self):
    """Adds the core library."""
    # Add externs and closure dependencies.
    source_base = shakaBuildHelpers.get_source_base()
    match = re.compile(r'.*\.js$')
    self.include |= set(
        shakaBuildHelpers.get_all_files(
            os.path.join(source_base, 'externs'), match) +
        shakaBuildHelpers.get_all_files(
            os.path.join(source_base, 'third_party', 'closure'), match))

    # Check that there are no files in 'core' that are removed
    core_build = Build()
    core_build.parse_build(['+@core'], os.getcwd())
    core_files = core_build.include
    if self.exclude & core_files:
      print >> sys.stderr, 'Cannot exclude files from core'
    self.include |= core_files

  def parse_build(self, lines, root):
    """Parses a Build object from the given lines of commands.

    This will recursively read and parse builds.

    Args:
      lines: An array of strings defining commands.
      root: The full path to the base directory.

    Returns:
      True on success, False otherwise.
    """
    for line in lines:
      # Strip comments
      try:
        line = line[:line.index('#')]
      except ValueError:
        pass

      # Strip whitespace and ignore empty lines.
      line = line.strip()
      if not line:
        continue

      if line[0] == '+':
        is_neg = False
        line = line[1:].strip()
      elif line[0] == '-':
        is_neg = True
        line = line[1:].strip()
      else:
        print >> sys.stderr, 'Operation (+/-) required'
        return False

      if line[0] == '@':
        line = line[1:].strip()

        build_path = self._get_build_file_path(line, root)
        if not build_path:
          return False
        lines = open(build_path).readlines()
        sub_root = os.path.dirname(build_path)

        # If this is a build file, then recurse and combine the builds.
        sub_build = Build()
        if not sub_build.parse_build(lines, sub_root):
          return False

        if is_neg:
          self._combine(sub_build.reverse())
        else:
          self._combine(sub_build)
      else:
        if not os.path.isabs(line):
          line = os.path.abspath(os.path.join(root, line))
        if not os.path.isfile(line):
          print >> sys.stderr, 'Unable to find file ' + line
          return False

        if is_neg:
          self.include.discard(line)
          self.exclude.add(line)
        else:
          self.include.add(line)
          self.exclude.discard(line)

    return True

  def build_raw(self, extra_opts, is_debug):
    """Builds the files in |self.include| using the given extra Closure options.

    Args:
      extra_opts: An array of extra options to give to Closure.
      is_debug: True to compile for debugging, false for release.

    Returns:
      True on success; False on failure.
    """
    jar = os.path.join(shakaBuildHelpers.get_source_base(),
                       'third_party', 'closure', 'compiler.jar')
    jar = shakaBuildHelpers.cygwin_safe_path(jar)
    files = [shakaBuildHelpers.cygwin_safe_path(f) for f in self.include]
    files.sort()

    try:
      if is_debug:
        closure_opts = common_closure_opts + debug_closure_opts
      else:
        closure_opts = common_closure_opts + release_closure_opts

      cmd_line = ['java', '-jar', jar] + closure_opts + extra_opts + files
      shakaBuildHelpers.print_cmd_line(cmd_line)
      subprocess.check_call(cmd_line)
      return True
    except subprocess.CalledProcessError:
      print >> sys.stderr, 'Build failed'
      return False

  def build_library(self, name, rebuild, is_debug):
    """Builds Shaka Player using the files in |self.include|.

    Args:
      name: The name of the build.
      rebuild: True to rebuild, False to ignore if no changes are detected.
      is_debug: True to compile for debugging, false for release.

    Returns:
      True on success; False on failure.
    """
    self.add_core()

    # In the build files, we use '/' in the paths, however Windows uses '\'.
    # Although Windows supports both, the source mapping will not work.  So
    # use Linux-style paths for arguments.
    source_base = shakaBuildHelpers.get_source_base().replace('\\', '/')
    if is_debug:
      name += '.debug'

    result_prefix = shakaBuildHelpers.cygwin_safe_path(
        os.path.join(source_base, 'dist', 'shaka-player.' + name))
    result_file = result_prefix + '.js'
    result_map = result_prefix + '.map'

    # Detect changes to the library and only build if changes have been made.
    if not rebuild and os.path.isfile(result_file):
      build_time = os.path.getmtime(result_file)
      complete_build = Build()
      if complete_build.parse_build(['+@complete'], os.getcwd()):
        complete_build.add_core()
        # Get a list of files modified since the build file was.
        edited_files = [f for f in complete_build.include
                        if os.path.getmtime(f) > build_time]
        if not edited_files:
          print 'No changes detected, not building.  Use --force to override.'
          return True

    opts = ['--create_source_map', result_map, '--js_output_file', result_file,
            '--source_map_location_mapping', source_base + '|..',
            '--dependency_mode=LOOSE', '--js=shaka-player.uncompiled.js']
    if not self.build_raw(opts, is_debug):
      return False

    # Add a special source-mapping comment so that Chrome and Firefox can map
    # line and character numbers from the compiled library back to the original
    # source locations.
    with open(result_file, 'a') as f:
      f.write('//# sourceMappingURL=shaka-player.' + name + '.map')

    return True

def usage():
  print 'Usage:', sys.argv[0], """[options] [commands]

Options:
 --debug          : Make a debug compiled file (e.g. don't rename internals).
 --force          : Build the library even if no changes are detected.
 --help           : Prints this help page.
 --name           : Sets the name of the build, uses 'compiled' if not given.
"""
  print __doc__

def main(args):
  name = 'compiled'
  lines = []
  rebuild = False
  is_debug = False
  i = 0
  while i < len(args):
    if args[i] == '--name':
      i += 1
      if i == len(args):
        print >> sys.stderr, '--name requires an argument'
        return 1
      name = args[i]
    elif args[i] == '--debug':
      is_debug = True
    elif args[i] == '--force':
      rebuild = True
    elif args[i] == '--help':
      usage()
      return 0
    elif args[i].startswith('--'):
      print >> sys.stderr, 'Unknown option', args[i]
      usage()
      return 1
    else:
      lines.append(args[i])
    i += 1

  if not lines:
    lines = ['+@complete']

  print 'Compiling the library...'
  custom_build = Build()
  if not custom_build.parse_build(lines, os.getcwd()):
    return 1
  return 0 if custom_build.build_library(name, rebuild, is_debug) else 1

if __name__ == '__main__':
  shakaBuildHelpers.run_main(main)

<commit_msg>Revert dependency_mode=loose<commit_after>#!/usr/bin/python
#
# Copyright 2016 Google Inc.  All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a build from the given commands.

A command is either an addition or a subtraction.  An addition is prefixed with
a +; a subtraction is when prefixed with a -.  After the character, there is a
name of a file or a @ sign and the name of a build file.

Build files are the files found in build/types.  These files are simply a
newline separated list of commands to execute.  So if the "+@complete" command
is given, it will open the complete file and run it (which may in turn open
other build files).  Subtracting a build file will reverse all actions applied
by the given file.  So "-@networking" will remove all the networking plugins.

The core library is always included so does not have to be listed.  The default
is to use the name 'compiled'; if no commands are given, it will build the
complete build.

Examples:
  # Equivalent to +@complete
  build.py

  build.py +@complete
  build.py +@complete -@networking
  build.py --name custom +@manifests +@networking +../my_plugin.js
"""

import os
import re
import shutil
import subprocess
import sys

import shakaBuildHelpers

common_closure_opts = [
    '--language_in', 'ECMASCRIPT5',
    '--language_out', 'ECMASCRIPT3',

    '--jscomp_error=*',

    # 'deprecatedAnnotations' controls complains about @expose, but the new
    # @nocollapse annotation does not do the same job for properties.
    # So since we can't use the new annotations, we have to ignore complaints
    # about the old one.
    '--jscomp_off=deprecatedAnnotations',

    # Analyzer checks require explicit nullability, which is a pain.
    '--jscomp_off=analyzerChecksInternal',

    '--extra_annotation_name=listens',
    '--extra_annotation_name=exportDoc',

    '--conformance_configs',
    ('%s/build/conformance.textproto' %
     shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())),

    '--generate_exports',

    '-D', 'COMPILED=true',
    '-D', 'goog.STRICT_MODE_COMPATIBLE=true',
    '-D', 'goog.ENABLE_DEBUG_LOADER=false',
    '-D', 'GIT_VERSION="%s"' % shakaBuildHelpers.calculate_version()
]
debug_closure_opts = [
    # Don't use a wrapper script in debug mode so all the internals are visible
    # on the global object.

    '-O', 'SIMPLE',
    '-D', 'goog.DEBUG=true',
    '-D', 'goog.asserts.ENABLE_ASSERTS=true',
    '-D', 'shaka.log.MAX_LOG_LEVEL=4',  # shaka.log.Level.DEBUG
]
release_closure_opts = [
    ('--output_wrapper_file=%s/build/wrapper.template.js' %
     shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())),

    '-O', 'ADVANCED',
    '-D', 'goog.DEBUG=false',
    '-D', 'goog.asserts.ENABLE_ASSERTS=false',
    '-D', 'shaka.log.MAX_LOG_LEVEL=0',
]

class Build(object):
  """Defines a build that has been parsed from a build file.

  This has exclude files even though it will not be used at the top-level.  This
  allows combining builds.  A file will only exist in at most one set.

  Members:
    include - A set of files to include.
    exclude - A set of files to remove.
  """

  def __init__(self, include=None, exclude=None):
    self.include = include or set()
    self.exclude = exclude or set()

  def _get_build_file_path(self, name, root):
    """Gets the full path to a build file, if it exists.

    Args:
      name: The string name to check.
      root: The full path to the base directory.

    Returns:
      The full path to the build file, or None if not found.
    """
    source_base = shakaBuildHelpers.get_source_base()
    local_path = os.path.join(root, name)
    build_path = os.path.join(source_base, 'build', 'types', name)
    if (os.path.isfile(local_path) and os.path.isfile(build_path)
        and local_path != build_path):
      print >> sys.stderr, 'Build file "%s" is ambiguous' % name
      return None
    elif os.path.isfile(local_path):
      return local_path
    elif os.path.isfile(build_path):
      return build_path
    else:
      print >> sys.stderr, 'Build file not found: ' + name
      return None

  def _combine(self, other):
    include_all = self.include | other.include
    exclude_all = self.exclude | other.exclude
    self.include = include_all - exclude_all
    self.exclude = exclude_all - include_all

  def reverse(self):
    return Build(self.exclude, self.include)

  def add_core(self):
    """Adds the core library."""
    # Add externs and closure dependencies.
    source_base = shakaBuildHelpers.get_source_base()
    match = re.compile(r'.*\.js$')
    self.include |= set(
        shakaBuildHelpers.get_all_files(
            os.path.join(source_base, 'externs'), match) +
        shakaBuildHelpers.get_all_files(
            os.path.join(source_base, 'third_party', 'closure'), match))

    # Check that there are no files in 'core' that are removed
    core_build = Build()
    core_build.parse_build(['+@core'], os.getcwd())
    core_files = core_build.include
    if self.exclude & core_files:
      print >> sys.stderr, 'Cannot exclude files from core'
    self.include |= core_files

  def parse_build(self, lines, root):
    """Parses a Build object from the given lines of commands.

    This will recursively read and parse builds.

    Args:
      lines: An array of strings defining commands.
      root: The full path to the base directory.

    Returns:
      True on success, False otherwise.
    """
    for line in lines:
      # Strip comments
      try:
        line = line[:line.index('#')]
      except ValueError:
        pass

      # Strip whitespace and ignore empty lines.
      line = line.strip()
      if not line:
        continue

      if line[0] == '+':
        is_neg = False
        line = line[1:].strip()
      elif line[0] == '-':
        is_neg = True
        line = line[1:].strip()
      else:
        print >> sys.stderr, 'Operation (+/-) required'
        return False

      if line[0] == '@':
        line = line[1:].strip()

        build_path = self._get_build_file_path(line, root)
        if not build_path:
          return False
        lines = open(build_path).readlines()
        sub_root = os.path.dirname(build_path)

        # If this is a build file, then recurse and combine the builds.
        sub_build = Build()
        if not sub_build.parse_build(lines, sub_root):
          return False

        if is_neg:
          self._combine(sub_build.reverse())
        else:
          self._combine(sub_build)
      else:
        if not os.path.isabs(line):
          line = os.path.abspath(os.path.join(root, line))
        if not os.path.isfile(line):
          print >> sys.stderr, 'Unable to find file ' + line
          return False

        if is_neg:
          self.include.discard(line)
          self.exclude.add(line)
        else:
          self.include.add(line)
          self.exclude.discard(line)

    return True

  def build_raw(self, extra_opts, is_debug):
    """Builds the files in |self.include| using the given extra Closure options.

    Args:
      extra_opts: An array of extra options to give to Closure.
      is_debug: True to compile for debugging, false for release.

    Returns:
      True on success; False on failure.
    """
    jar = os.path.join(shakaBuildHelpers.get_source_base(),
                       'third_party', 'closure', 'compiler.jar')
    jar = shakaBuildHelpers.cygwin_safe_path(jar)
    files = [shakaBuildHelpers.cygwin_safe_path(f) for f in self.include]
    files.sort()

    try:
      if is_debug:
        closure_opts = common_closure_opts + debug_closure_opts
      else:
        closure_opts = common_closure_opts + release_closure_opts

      cmd_line = ['java', '-jar', jar] + closure_opts + extra_opts + files
      shakaBuildHelpers.print_cmd_line(cmd_line)
      subprocess.check_call(cmd_line)
      return True
    except subprocess.CalledProcessError:
      print >> sys.stderr, 'Build failed'
      return False

  def build_library(self, name, rebuild, is_debug):
    """Builds Shaka Player using the files in |self.include|.

    Args:
      name: The name of the build.
      rebuild: True to rebuild, False to ignore if no changes are detected.
      is_debug: True to compile for debugging, false for release.

    Returns:
      True on success; False on failure.
    """
    self.add_core()

    # In the build files, we use '/' in the paths, however Windows uses '\'.
    # Although Windows supports both, the source mapping will not work.  So
    # use Linux-style paths for arguments.
    source_base = shakaBuildHelpers.get_source_base().replace('\\', '/')
    if is_debug:
      name += '.debug'

    result_prefix = shakaBuildHelpers.cygwin_safe_path(
        os.path.join(source_base, 'dist', 'shaka-player.' + name))
    result_file = result_prefix + '.js'
    result_map = result_prefix + '.map'

    # Detect changes to the library and only build if changes have been made.
    if not rebuild and os.path.isfile(result_file):
      build_time = os.path.getmtime(result_file)
      complete_build = Build()
      if complete_build.parse_build(['+@complete'], os.getcwd()):
        complete_build.add_core()
        # Get a list of files modified since the build file was.
        edited_files = [f for f in complete_build.include
                        if os.path.getmtime(f) > build_time]
        if not edited_files:
          print 'No changes detected, not building.  Use --force to override.'
          return True

    opts = ['--create_source_map', result_map, '--js_output_file', result_file,
            '--source_map_location_mapping', source_base + '|..']
    if not self.build_raw(opts, is_debug):
      return False

    # Add a special source-mapping comment so that Chrome and Firefox can map
    # line and character numbers from the compiled library back to the original
    # source locations.
    with open(result_file, 'a') as f:
      f.write('//# sourceMappingURL=shaka-player.' + name + '.map')

    return True

def usage():
  print 'Usage:', sys.argv[0], """[options] [commands]

Options:
 --debug          : Make a debug compiled file (e.g. don't rename internals).
 --force          : Build the library even if no changes are detected.
 --help           : Prints this help page.
 --name           : Sets the name of the build, uses 'compiled' if not given.
"""
  print __doc__

def main(args):
  name = 'compiled'
  lines = []
  rebuild = False
  is_debug = False
  i = 0
  while i < len(args):
    if args[i] == '--name':
      i += 1
      if i == len(args):
        print >> sys.stderr, '--name requires an argument'
        return 1
      name = args[i]
    elif args[i] == '--debug':
      is_debug = True
    elif args[i] == '--force':
      rebuild = True
    elif args[i] == '--help':
      usage()
      return 0
    elif args[i].startswith('--'):
      print >> sys.stderr, 'Unknown option', args[i]
      usage()
      return 1
    else:
      lines.append(args[i])
    i += 1

  if not lines:
    lines = ['+@complete']

  print 'Compiling the library...'
  custom_build = Build()
  if not custom_build.parse_build(lines, os.getcwd()):
    return 1
  return 0 if custom_build.build_library(name, rebuild, is_debug) else 1

if __name__ == '__main__':
  shakaBuildHelpers.run_main(main)

<|endoftext|>


<commit_before>'''
LDTP v2 utils.

@author: Eitan Isaacson <eitan@ascender.com>
@author: Nagappan Alagappan <nagappan@gmail.com>
@copyright: Copyright (c) 2009 Eitan Isaacson
@copyright: Copyright (c) 2009 Nagappan Alagappan
@license: LGPL

http://ldtp.freedesktop.org

This file may be distributed and/or modified under the terms of the GNU General
Public License version 2 as published by the Free Software Foundation. This file
is distributed without any warranty; without even the implied warranty of
merchantability or fitness for a particular purpose.

See "COPYING" in the source distribution for more information.

Headers in this file shall remain intact.
'''

import re
import pyatspi
import orderedset
from re import match as re_match
from constants import abbreviated_roles
from fnmatch import translate as glob_trans
from server_exception import LdtpServerException

class Utils:
    cached_apps = None
    def __init__(self):
        lazy_load = True
        self._desktop = pyatspi.Registry.getDesktop(0)
        if Utils.cached_apps is None:
            pyatspi.Registry.registerEventListener(
                self._on_window_event, 'window')
            Utils.cached_apps = orderedset.OrderedSet()
            if lazy_load:
                for app in self._desktop:
                    if app is None: continue
                    self.cached_apps.add(app)

    def _on_window_event(self, event):
        self.cached_apps.add(event.host_application)

    def _list_apps(self):
        for app in list(self.cached_apps):
            if not app: continue
            yield app

    def _list_guis(self):
        for app in list(self.cached_apps):
            if not app: continue
            try:
                for gui in app:
                    if not gui: continue
                    yield gui
            except LookupError:
                self.cached_apps.discard(app)

    def _ldtpize_accessible(self, acc):
        label_acc = None
        rel_set = acc.getRelationSet()
        if rel_set:
            for i, rel in enumerate(rel_set):
                relationType = rel.getRelationType()
                if relationType == pyatspi.RELATION_LABELLED_BY or \
                        relationType == pyatspi.RELATION_CONTROLLED_BY:
                    label_acc = rel.getTarget(i)
                    break
        return abbreviated_roles.get(acc.getRole(), 'ukn'), \
            (label_acc or acc).name.replace(' ', '').rstrip(':.')

    def _glob_match(self, pattern, string):
        return bool(re_match(glob_trans(pattern), string))

    def _match_name_to_acc(self, name, acc):
        if acc.name == name:
            return 1
        _ldtpize_accessible_name = self._ldtpize_accessible(acc)
        _object_name = '%s%s' % (_ldtpize_accessible_name[0],
                                 _ldtpize_accessible_name[1])
        if _object_name  == name:
            return 1
        if self._glob_match(name, acc.name):
            return 1
        if self._glob_match(name, _object_name):
            return 1
        role = acc.getRole()
        if role == pyatspi.ROLE_FRAME or role == pyatspi.ROLE_DIALOG or \
                role == pyatspi.ROLE_WINDOW or \
                role == pyatspi.ROLE_FONT_CHOOSER or \
                role == pyatspi.ROLE_FILE_CHOOSER or \
                role == pyatspi.ROLE_ALERT or \
                role == pyatspi.ROLE_COLOR_CHOOSER:
            strip = '( |\n)'
        else:
            strip = '( |:|\.|_|\n)'
        _tmp_name = re.sub(strip, '', name)
        if self._glob_match(_tmp_name, _object_name):
            return 1
        if self._glob_match(_tmp_name, _ldtpize_accessible_name[1]):
            return 1
        return 0

    def _match_name_to_appmap(self, name, appmap_name):
        """
        Required when object name has empty label
        """
        if name == appmap_name:
            return 1
        if self._glob_match(name, appmap_name):
            return 1
        return 0

    def _list_objects(self, obj):
        if obj:
            yield obj
            for child in obj:
                for c in self._list_objects(child):
                    yield c

    def _get_combo_child_object_type(self, obj):
        """
        This function will check for all levels and returns the first
        matching LIST / MENU type
        """
        if obj:
            for child in obj:
                if not child:
                    continue
                if child.childCount > 0:
                    child_obj = self._get_combo_child_object_type(child)
                    if child_obj:
                        return child_obj
                if child.getRole() == pyatspi.ROLE_LIST:
                    return child
                elif child.getRole() == pyatspi.ROLE_MENU:
                    return child

    def _get_child_object_type(self, obj, role_type):
        """
        This function will check for all levels and returns the first
        matching role_type
        """
        if obj and role_type:
            for child in obj:
                if not child:
                    continue
                if child.childCount > 0:
                    child_obj = self._get_child_object_type(child, role_type)
                    if child_obj:
                        return child_obj
                if child.getRole() == role_type:
                    return child

    def _appmap_pairs(self, gui):
        ldtpized_list = []
        ldtpized_obj_index = {}
        for obj in self._list_objects(gui):
            abbrev_role, abbrev_name = self._ldtpize_accessible(obj)
            if abbrev_role in ldtpized_obj_index:
                ldtpized_obj_index[abbrev_role] += 1
            else:
                ldtpized_obj_index[abbrev_role] = 0
            if abbrev_name == '':
                ldtpized_name_base = abbrev_role
                ldtpized_name = '%s%d' % (ldtpized_name_base,
                                          ldtpized_obj_index[abbrev_role])
            else:
                ldtpized_name_base = '%s%s' % (abbrev_role, abbrev_name)
                ldtpized_name = ldtpized_name_base
            i = 1
            while ldtpized_name in ldtpized_list:
                ldtpized_name = '%s%d' % (ldtpized_name_base,
                                          i)
                i += 1
            ldtpized_list.append(ldtpized_name)
            yield ldtpized_name, obj, '%s#%d' % (abbrev_role,
                                                 ldtpized_obj_index[abbrev_role])

    def _get_menu_hierarchy(self, window_name, object_name):
        _menu_hierarchy = re.split(';', object_name)
        # Get handle of menu
        obj = self._get_object(window_name, _menu_hierarchy[0])
        # Navigate all sub-menu under a menu
        for _menu in _menu_hierarchy[1:]:
            _flag = False
            for _child in self._list_objects(obj):
                if obj == _child:
                    # if the given object and child object matches
                    continue
                if self._match_name_to_acc(_menu, _child):
                    obj = _child
                    _flag = True
                    break
            if not _flag:
                raise LdtpServerException (
                    "Menu item %s doesn't exist in hierarchy" % _menu)
        return obj

    def _click_object(self, obj, action = 'click'):
        try:
            iaction = obj.queryAction()
        except NotImplementedError:
            raise LdtpServerException(
                'Object does not have an Action interface')
        else:
            for i in xrange(iaction.nActions):
                if re.match(action, iaction.getName(i)):
                    iaction.doAction(i)
                    return
            raise LdtpServerException('Object does not have a "click" action')

    def _get_window_handle(self, window_name):
        window_list = []
        window_type = {}

        # Search with accessible name
        for gui in self._list_guis():
            if self._match_name_to_acc(window_name, gui):
                return gui

        # Search with LDTP appmap format
        for gui in self._list_guis():
            w_name = self._ldtpize_accessible(gui)
            if w_name[1] == '':
                if w_name[0] in window_type:
                    window_type[w_name[0]] += 1
                else:
                    window_type[w_name[0]] = 0
                tmp_name = '%d' % window_type[w_name[0]]
            else:
                tmp_name = w_name[1]
            w_name = tmp_name = '%s%s' % (w_name[0], tmp_name)
            index = 1
            while w_name in window_list:
                w_name = '%s%d' % (tmp_name, index)
                index += 1
            window_list.append(w_name)
            if window_name == w_name:
                return gui
            if self._glob_match(window_name, w_name):
                return gui
            if self._glob_match(window_name, w_name):
                return gui
            if self._glob_match(re.sub(' ', '', window_name),
                                re.sub(' ', '', w_name)):
                return gui
        return None

    def _get_object(self, window_name, obj_name):
        _window_handle = self._get_window_handle(window_name)
        if not _window_handle:
            raise LdtpServerException('Unable to find window "%s"' % \
                                          window_name)
        for name, obj, obj_index in self._appmap_pairs(_window_handle):
            if self._glob_match(obj_name, obj_index) or \
                    self._match_name_to_acc(obj_name, obj) or \
                    self._match_name_to_appmap(obj_name, name):
                return obj
        raise LdtpServerException(
            'Unable to find object name "%s" in application map' % obj_name)

    def _grab_focus(self, obj):
        try:
            componenti = obj.queryComponent()
        except:
            raise LdtpServerException('Failed to grab focus for %s' % obj)
        componenti.grabFocus()

    def _get_accessible_at_row_column(self, obj, row_index, column_index):
        try:
            tablei = obj.queryTable()
        except NotImplementedError:
            raise LdtpServerException('Object not table type.')

        if row_index < 0 or row_index > tablei.nRows:
            raise LdtpServerException('Row index out of range: %d' % row_index)

        if column_index < 0 or column_index > tablei.nColumns:
            raise LdtpServerException('Column index out of range: %d' % \
                                          column_index)

        cell = tablei.getAccessibleAt(row_index, column_index)
        if not cell:
            raise LdtpServerException('Unable to access table cell on ' \
                                          'the given row and column index')
        return cell

    def _check_state(self, obj, object_state):
        _state = obj.getState()
        _current_state = _state.getStates()

        _status = False
        if object_state in _current_state:
            _status = True

        return _status

    def _mouse_event(self, x, y, name = 'b1c'):
        pyatspi.Registry.generateMouseEvent(x, y, name)

        return 1

    def _get_size(self, obj):
        try:
            componenti = obj.queryComponent()
        except:
            raise LdtpServerException('Failed to grab focus for %s' % obj)
        return componenti.getExtents(pyatspi.DESKTOP_COORDS)
<commit_msg>Fixes unicode search<commit_after>'''
LDTP v2 utils.

@author: Eitan Isaacson <eitan@ascender.com>
@author: Nagappan Alagappan <nagappan@gmail.com>
@copyright: Copyright (c) 2009 Eitan Isaacson
@copyright: Copyright (c) 2009 Nagappan Alagappan
@license: LGPL

http://ldtp.freedesktop.org

This file may be distributed and/or modified under the terms of the GNU General
Public License version 2 as published by the Free Software Foundation. This file
is distributed without any warranty; without even the implied warranty of
merchantability or fitness for a particular purpose.

See "COPYING" in the source distribution for more information.

Headers in this file shall remain intact.
'''

import re
import pyatspi
import orderedset
from re import match as re_match
from constants import abbreviated_roles
from fnmatch import translate as glob_trans
from server_exception import LdtpServerException

class Utils:
    cached_apps = None
    def __init__(self):
        lazy_load = True
        self._desktop = pyatspi.Registry.getDesktop(0)
        if Utils.cached_apps is None:
            pyatspi.Registry.registerEventListener(
                self._on_window_event, 'window')
            Utils.cached_apps = orderedset.OrderedSet()
            if lazy_load:
                for app in self._desktop:
                    if app is None: continue
                    self.cached_apps.add(app)

    def _on_window_event(self, event):
        self.cached_apps.add(event.host_application)

    def _list_apps(self):
        for app in list(self.cached_apps):
            if not app: continue
            yield app

    def _list_guis(self):
        for app in list(self.cached_apps):
            if not app: continue
            try:
                for gui in app:
                    if not gui: continue
                    yield gui
            except LookupError:
                self.cached_apps.discard(app)

    def _ldtpize_accessible(self, acc):
        label_acc = None
        rel_set = acc.getRelationSet()
        if rel_set:
            for i, rel in enumerate(rel_set):
                relationType = rel.getRelationType()
                if relationType == pyatspi.RELATION_LABELLED_BY or \
                        relationType == pyatspi.RELATION_CONTROLLED_BY:
                    label_acc = rel.getTarget(i)
                    break
        return abbreviated_roles.get(acc.getRole(), 'ukn'), \
            (label_acc or acc).name.replace(' ', '').rstrip(':.')

    def _glob_match(self, pattern, string):
        return bool(re_match(glob_trans(pattern), string, re.M | re.U | re.L))

    def _match_name_to_acc(self, name, acc):
        if acc.name == name:
            return 1
        _ldtpize_accessible_name = self._ldtpize_accessible(acc)
        _object_name = u'%s%s' % (_ldtpize_accessible_name[0],
                                  _ldtpize_accessible_name[1])
        if _object_name  == name:
            return 1
        if self._glob_match(name, acc.name):
            return 1
        if self._glob_match(name, _object_name):
            return 1
        role = acc.getRole()
        if role == pyatspi.ROLE_FRAME or role == pyatspi.ROLE_DIALOG or \
                role == pyatspi.ROLE_WINDOW or \
                role == pyatspi.ROLE_FONT_CHOOSER or \
                role == pyatspi.ROLE_FILE_CHOOSER or \
                role == pyatspi.ROLE_ALERT or \
                role == pyatspi.ROLE_COLOR_CHOOSER:
            strip = '( |\n)'
        else:
            strip = '( |:|\.|_|\n)'
        _tmp_name = re.sub(strip, '', name)
        if self._glob_match(_tmp_name, _object_name):
            return 1
        if self._glob_match(_tmp_name, _ldtpize_accessible_name[1]):
            return 1
        return 0

    def _match_name_to_appmap(self, name, appmap_name):
        """
        Required when object name has empty label
        """
        if name == appmap_name:
            return 1
        if self._glob_match(name, appmap_name):
            return 1
        return 0

    def _list_objects(self, obj):
        if obj:
            yield obj
            for child in obj:
                for c in self._list_objects(child):
                    yield c

    def _get_combo_child_object_type(self, obj):
        """
        This function will check for all levels and returns the first
        matching LIST / MENU type
        """
        if obj:
            for child in obj:
                if not child:
                    continue
                if child.childCount > 0:
                    child_obj = self._get_combo_child_object_type(child)
                    if child_obj:
                        return child_obj
                if child.getRole() == pyatspi.ROLE_LIST:
                    return child
                elif child.getRole() == pyatspi.ROLE_MENU:
                    return child

    def _get_child_object_type(self, obj, role_type):
        """
        This function will check for all levels and returns the first
        matching role_type
        """
        if obj and role_type:
            for child in obj:
                if not child:
                    continue
                if child.childCount > 0:
                    child_obj = self._get_child_object_type(child, role_type)
                    if child_obj:
                        return child_obj
                if child.getRole() == role_type:
                    return child

    def _appmap_pairs(self, gui):
        ldtpized_list = []
        ldtpized_obj_index = {}
        for obj in self._list_objects(gui):
            abbrev_role, abbrev_name = self._ldtpize_accessible(obj)
            if abbrev_role in ldtpized_obj_index:
                ldtpized_obj_index[abbrev_role] += 1
            else:
                ldtpized_obj_index[abbrev_role] = 0
            if abbrev_name == '':
                ldtpized_name_base = abbrev_role
                ldtpized_name = u'%s%d' % (ldtpized_name_base,
                                           ldtpized_obj_index[abbrev_role])
            else:
                ldtpized_name_base = u'%s%s' % (abbrev_role, abbrev_name)
                ldtpized_name = ldtpized_name_base
            i = 1
            while ldtpized_name in ldtpized_list:
                ldtpized_name = u'%s%d' % (ldtpized_name_base,
                                           i)
                i += 1
            ldtpized_list.append(ldtpized_name)
            yield ldtpized_name, obj, u'%s#%d' % (abbrev_role,
                                                  ldtpized_obj_index[abbrev_role])

    def _get_menu_hierarchy(self, window_name, object_name):
        _menu_hierarchy = re.split(';', object_name)
        # Get handle of menu
        obj = self._get_object(window_name, _menu_hierarchy[0])
        # Navigate all sub-menu under a menu
        for _menu in _menu_hierarchy[1:]:
            _flag = False
            for _child in self._list_objects(obj):
                if obj == _child:
                    # if the given object and child object matches
                    continue
                if self._match_name_to_acc(_menu, _child):
                    obj = _child
                    _flag = True
                    break
            if not _flag:
                raise LdtpServerException (
                    "Menu item %s doesn't exist in hierarchy" % _menu)
        return obj

    def _click_object(self, obj, action = 'click'):
        try:
            iaction = obj.queryAction()
        except NotImplementedError:
            raise LdtpServerException(
                'Object does not have an Action interface')
        else:
            for i in xrange(iaction.nActions):
                if re.match(action, iaction.getName(i)):
                    iaction.doAction(i)
                    return
            raise LdtpServerException('Object does not have a "click" action')

    def _get_window_handle(self, window_name):
        window_list = []
        window_type = {}

        # Search with accessible name
        for gui in self._list_guis():
            if self._match_name_to_acc(window_name, gui):
                return gui

        # Search with LDTP appmap format
        for gui in self._list_guis():
            w_name = self._ldtpize_accessible(gui)
            if w_name[1] == '':
                if w_name[0] in window_type:
                    window_type[w_name[0]] += 1
                else:
                    window_type[w_name[0]] = 0
                tmp_name = '%d' % window_type[w_name[0]]
            else:
                tmp_name = w_name[1]
            w_name = tmp_name = u'%s%s' % (w_name[0], tmp_name)
            index = 1
            while w_name in window_list:
                w_name = u'%s%d' % (tmp_name, index)
                index += 1
            window_list.append(w_name)
            if window_name == w_name:
                return gui
            if self._glob_match(window_name, w_name):
                return gui
            if self._glob_match(window_name, w_name):
                return gui
            if self._glob_match(re.sub(' ', '', window_name),
                                re.sub(' ', '', w_name)):
                return gui
        return None

    def _get_object(self, window_name, obj_name):
        _window_handle = self._get_window_handle(window_name)
        if not _window_handle:
            raise LdtpServerException('Unable to find window "%s"' % \
                                          window_name)
        for name, obj, obj_index in self._appmap_pairs(_window_handle):
            if self._glob_match(obj_name, obj_index) or \
                    self._match_name_to_acc(obj_name, obj) or \
                    self._match_name_to_appmap(obj_name, name):
                return obj
        raise LdtpServerException(
            'Unable to find object name "%s" in application map' % obj_name)

    def _grab_focus(self, obj):
        try:
            componenti = obj.queryComponent()
        except:
            raise LdtpServerException('Failed to grab focus for %s' % obj)
        componenti.grabFocus()

    def _get_accessible_at_row_column(self, obj, row_index, column_index):
        try:
            tablei = obj.queryTable()
        except NotImplementedError:
            raise LdtpServerException('Object not table type.')

        if row_index < 0 or row_index > tablei.nRows:
            raise LdtpServerException('Row index out of range: %d' % row_index)

        if column_index < 0 or column_index > tablei.nColumns:
            raise LdtpServerException('Column index out of range: %d' % \
                                          column_index)

        cell = tablei.getAccessibleAt(row_index, column_index)
        if not cell:
            raise LdtpServerException('Unable to access table cell on ' \
                                          'the given row and column index')
        return cell

    def _check_state(self, obj, object_state):
        _state = obj.getState()
        _current_state = _state.getStates()

        _status = False
        if object_state in _current_state:
            _status = True

        return _status

    def _mouse_event(self, x, y, name = 'b1c'):
        pyatspi.Registry.generateMouseEvent(x, y, name)

        return 1

    def _get_size(self, obj):
        try:
            componenti = obj.queryComponent()
        except:
            raise LdtpServerException('Failed to grab focus for %s' % obj)
        return componenti.getExtents(pyatspi.DESKTOP_COORDS)
<|endoftext|>


<commit_before>from __future__ import with_statement

import json
import logging
import os
from datetime import datetime

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sitemaps import GenericSitemap
from django.core.mail import EmailMessage, mail_admins, send_mail
from django.db import connection, transaction
from django.template.loader import render_to_string
from django.utils.encoding import smart_str

from celery import task, chord
from constance import config
from lxml import etree

from kuma.core.cache import memcache
from kuma.core.utils import chord_flow, chunked, MemcacheLock
from kuma.search.models import Index

from .events import context_dict
from .exceptions import PageMoveError, StaleDocumentsRenderingInProgress
from .helpers import absolutify
from .models import Document, Revision, RevisionIP
from .search import WikiDocumentType
from .utils import tidy_content

log = logging.getLogger('kuma.wiki.tasks')
render_lock = MemcacheLock('render-stale-documents-lock', expires=60 * 60)

@task(rate_limit='60/m')
def render_document(pk, cache_control, base_url, force=False):
    """Simple task wrapper for the render() method of the Document model"""
    document = Document.objects.get(pk=pk)
    if force:
        document.render_started_at = None

    try:
        document.render(cache_control, base_url)
    except Exception as e:
        subject = 'Exception while rendering document %s' % document.pk
        mail_admins(subject=subject, message=e)
    return document.rendered_errors

@task
def email_render_document_progress(percent_complete, total):
    """
    Task to send email for render_document progress notification.
    """
    subject = ('The command `render_document` is %s%% complete' %
               percent_complete)
    message = (
        'The command `render_document` is %s%% complete out of a total of '
        '%s documents to render.' % (percent_complete, total))
    mail_admins(subject=subject, message=message)

@task
def render_document_chunk(pks, cache_control='no-cache', base_url=None,
                          force=False):
    """
    Simple task to render a chunk of documents instead of one per each
    """
    logger = render_document_chunk.get_logger()
    logger.info(u'Starting to render document chunk: %s' %
                ','.join([str(pk) for pk in pks]))
    base_url = base_url or settings.SITE_URL
    for pk in pks:
        # calling the task without delay here since we want to localize
        # the processing of the chunk in one process
        result = render_document(pk, cache_control, base_url, force=force)
        if result:
            logger.error(u'Error while rendering document %s with error: %s' %
                         (pk, result))
    logger.info(u'Finished rendering of document chunk')

@task(throws=(StaleDocumentsRenderingInProgress,))
def acquire_render_lock():
    """
    A task to acquire the render document lock
    """
    if render_lock.locked():
        # fail loudly if this is running already
        # may indicate a problem with the schedule of this task
        raise StaleDocumentsRenderingInProgress
    render_lock.acquire()

@task
def release_render_lock():
    """
    A task to release the render document lock
    """
    render_lock.release()

@task
def render_stale_documents(log=None):
    """Simple task wrapper for rendering stale documents"""
    stale_docs = Document.objects.get_by_stale_rendering().distinct()
    stale_docs_count = stale_docs.count()
    if stale_docs_count == 0:
        # not stale documents to render
        return

    if log is None:
        # fetch a logger in case none is given
        log = render_stale_documents.get_logger()

    log.info('Found %s stale documents' % stale_docs_count)
    stale_pks = stale_docs.values_list('pk', flat=True)

    pre_task = acquire_render_lock.si()
    render_tasks = [render_document_chunk.si(pks)
                    for pks in chunked(stale_pks, 5)]
    post_task = release_render_lock.si()

    chord_flow(pre_task, render_tasks, post_task).apply_async()

@task
def build_json_data_for_document(pk, stale):
    """Force-refresh cached JSON data after rendering."""
    document = Document.objects.get(pk=pk)
    document.get_json_data(stale=stale)

    # If we're a translation, rebuild our source doc's JSON so its
    # translation list includes our last edit date.
    if document.parent is not None:
        parent_json = json.dumps(document.parent.build_json_data())
        Document.objects.filter(pk=document.parent.pk).update(json=parent_json)

@task
def move_page(locale, slug, new_slug, email):
    transaction.set_autocommit(False)
    User = get_user_model()
    try:
        user = User.objects.get(email=email)
    except User.DoesNotExist:
        transaction.rollback()
        logging.error('Page move failed: no user with email address %s' %
                      email)
        return
    try:
        doc = Document.objects.get(locale=locale, slug=slug)
    except Document.DoesNotExist:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale
    %(locale)s, but no such document exists.
    """ % {'slug': slug, 'locale': locale}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return
    try:
        doc._move_tree(new_slug, user=user)
    except PageMoveError as e:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale
    %(locale)s, but could not be completed.

    Diagnostic info:

    %(message)s
    """ % {'slug': slug, 'locale': locale, 'message': e.message}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return
    except Exception as e:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale %(locale)s,
    but could not be completed.

    %(info)s
    """ % {'slug': slug, 'locale': locale, 'info': e}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return

    transaction.commit()
    transaction.set_autocommit(True)

    # Now that we know the move succeeded, re-render the whole tree.
    for moved_doc in [doc] + doc.get_descendants():
        moved_doc.schedule_rendering('max-age=0')

    subject = 'Page move completed: ' + slug + ' (' + locale + ')'
    full_url = settings.SITE_URL + '/' + locale + '/docs/' + new_slug
    message = """
Page move completed.

The move requested for the document with slug %(slug)s in locale
%(locale)s, and all its children, has been completed.

You can now view this document at its new location: %(full_url)s.
    """ % {'slug': slug, 'locale': locale, 'full_url': full_url}
    send_mail(subject, message, settings.DEFAULT_FROM_EMAIL,
              [user.email])

@task
def update_community_stats():
    cursor = connection.cursor()
    try:
        cursor.execute("""
            SELECT count(creator_id)
            FROM
              (SELECT DISTINCT creator_id
               FROM wiki_revision
               WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
            """)
        contributors = cursor.fetchone()

        cursor.execute("""
            SELECT count(locale)
            FROM
              (SELECT DISTINCT wd.locale
               FROM wiki_document wd,
                                  wiki_revision wr
               WHERE wd.id = wr.document_id
                 AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
            """)
        locales = cursor.fetchone()
    finally:
        cursor.close()

    community_stats = {}

    try:
        community_stats['contributors'] = contributors[0]
        community_stats['locales'] = locales[0]
    except IndexError:
        community_stats = None

    # storing a None value in cache allows a better check for
    # emptiness in the view
    if 0 in community_stats.values():
        community_stats = None

    memcache.set('community_stats', community_stats, 86400)

@task
def delete_old_revision_ips(days=30):
    RevisionIP.objects.delete_old(days=days)

@task
def send_first_edit_email(revision_pk):
    """ Make an 'edited' notification email for first-time editors """
    revision = Revision.objects.get(pk=revision_pk)
    user, doc = revision.creator, revision.document
    subject = (u"[MDN] %(user)s made their first edit, to: %(doc)s" %
               {'user': user.username, 'doc': doc.title})
    message = render_to_string('wiki/email/edited.ltxt',
                               context_dict(revision))
    doc_url = absolutify(doc.get_absolute_url())
    email = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL,
                         to=[config.EMAIL_LIST_FOR_FIRST_EDITS],
                         headers={'X-Kuma-Document-Url': doc_url,
                                  'X-Kuma-Editor-Username': user.username})
    email.send()

class WikiSitemap(GenericSitemap):
    protocol = 'https'
    priority = 0.5

SITEMAP_START = u'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
SITEMAP_ELEMENT = u'<sitemap><loc>%s</loc><lastmod>%s</lastmod></sitemap>'
SITEMAP_END = u'</sitemapindex>'

@task
def build_locale_sitemap(locale):
    """
    For the given locale build the appropriate sitemap file and
    returns the locale, the file names written and timestamp of the
    build.
    """
    now = datetime.utcnow()
    timestamp = '%s+00:00' % now.replace(microsecond=0).isoformat()

    directory = os.path.join(settings.MEDIA_ROOT, 'sitemaps', locale)
    if not os.path.isdir(directory):
        os.makedirs(directory)

    queryset = Document.objects.filter_for_list(locale=locale)
    if queryset.exists():
        names = []
        info = {
            'queryset': queryset,
            'date_field': 'modified',
        }
        sitemap = WikiSitemap(info)
        for page in range(1, sitemap.paginator.num_pages + 1):
            urls = sitemap.get_urls(page=page)
            if page == 1:
                name = 'sitemap.xml'
            else:
                name = 'sitemap_%s.xml' % page
            names.append(name)

            rendered = smart_str(render_to_string('wiki/sitemap.xml',
                                                  {'urls': urls}))
            path = os.path.join(directory, name)
            with open(path, 'w') as sitemap_file:
                sitemap_file.write(rendered)

        return locale, names, timestamp

@task
def build_index_sitemap(results):
    """
    A chord callback task that writes a sitemap index file for the
    given results of :func:`~kuma.wiki.tasks.build_locale_sitemap` task.
    """
    sitemap_parts = [SITEMAP_START]

    for result in results:
        # result can be empty if no documents were found
        if result is not None:
            locale, names, timestamp = result
            for name in names:
                sitemap_url = absolutify('/sitemaps/%s/%s' % (locale, name))
                sitemap_parts.append(SITEMAP_ELEMENT % (sitemap_url, timestamp))

    sitemap_parts.append(SITEMAP_END)

    index_path = os.path.join(settings.MEDIA_ROOT, 'sitemap.xml')
    sitemap_tree = etree.fromstringlist(sitemap_parts)
    with open(index_path, 'w') as index_file:
        sitemap_tree.getroottree().write(index_file,
                                         encoding='utf-8',
                                         pretty_print=True)

@task
def build_sitemaps():
    """
    Build and save sitemap files for every MDN language and as a
    callback save the sitempa index file as well.
    """
    tasks = [build_locale_sitemap.si(locale)
             for locale in settings.MDN_LANGUAGES]
    post_task = build_index_sitemap.s()
    chord(header=tasks, body=post_task).apply_async()

@task
def index_documents(ids, index_pk, reraise=False):
    """
    Index a list of documents into the provided index.

    :arg ids: Iterable of `Document` pks to index.
    :arg index_pk: The `Index` pk of the index to index into.
    :arg reraise: False if you want errors to be swallowed and True
        if you want errors to be thrown.

    .. Note::

       This indexes all the documents in the chunk in one single bulk
       indexing call. Keep that in mind when you break your indexing
       task into chunks.

    """
    from kuma.wiki.models import Document

    cls = WikiDocumentType
    es = cls.get_connection('indexing')
    index = Index.objects.get(pk=index_pk)

    objects = Document.objects.filter(id__in=ids)
    documents = []
    for obj in objects:
        try:
            documents.append(cls.from_django(obj))
        except Exception:
            log.exception('Unable to extract/index document (id: %d)', obj.id)
            if reraise:
                raise

    if documents:
        cls.bulk_index(documents, id_field='id', es=es,
                       index=index.prefixed_name)

@task
def unindex_documents(ids, index_pk):
    """
    Delete a list of documents from the provided index.

    :arg ids: Iterable of `Document` pks to remove.
    :arg index_pk: The `Index` pk of the index to remove items from.
    """
    cls = WikiDocumentType
    es = cls.get_connection('indexing')
    index = Index.objects.get(pk=index_pk)

    cls.bulk_delete(ids, es=es, index=index.prefixed_name)

@task(rate_limit='120/m')
def tidy_revision_content(pk):
    """
    Run tidy over the given revision's content and save it to the
    tidy_content field if the content is not equal to the current value.

    :arg pk: Primary key of `Revision` whose content needs tidying.
    """
    revision = Revision.objects.get(pk=pk)
    tidied_content, errors = tidy_content(revision.content)
    if tidied_content != revision.tidied_content:
        Revision.objects.filter(pk=pk).update(tidied_content=tidied_content)
    # return the errors so we can look them up in the Celery task result store
    return errors
<commit_msg>bug 1174209 - catch, log, and retry DNE exceptions<commit_after>from __future__ import with_statement

import json
import logging
import os
from datetime import datetime

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sitemaps import GenericSitemap
from django.core.mail import EmailMessage, mail_admins, send_mail
from django.db import connection, transaction
from django.template.loader import render_to_string
from django.utils.encoding import smart_str

from celery import task, chord
from constance import config
from lxml import etree

from kuma.core.cache import memcache
from kuma.core.utils import chord_flow, chunked, MemcacheLock
from kuma.search.models import Index

from .events import context_dict
from .exceptions import PageMoveError, StaleDocumentsRenderingInProgress
from .helpers import absolutify
from .models import Document, Revision, RevisionIP
from .search import WikiDocumentType
from .utils import tidy_content

log = logging.getLogger('kuma.wiki.tasks')
render_lock = MemcacheLock('render-stale-documents-lock', expires=60 * 60)

@task(rate_limit='60/m')
def render_document(pk, cache_control, base_url, force=False):
    """Simple task wrapper for the render() method of the Document model"""
    document = Document.objects.get(pk=pk)
    if force:
        document.render_started_at = None

    try:
        document.render(cache_control, base_url)
    except Exception as e:
        subject = 'Exception while rendering document %s' % document.pk
        mail_admins(subject=subject, message=e)
    return document.rendered_errors

@task
def email_render_document_progress(percent_complete, total):
    """
    Task to send email for render_document progress notification.
    """
    subject = ('The command `render_document` is %s%% complete' %
               percent_complete)
    message = (
        'The command `render_document` is %s%% complete out of a total of '
        '%s documents to render.' % (percent_complete, total))
    mail_admins(subject=subject, message=message)

@task
def render_document_chunk(pks, cache_control='no-cache', base_url=None,
                          force=False):
    """
    Simple task to render a chunk of documents instead of one per each
    """
    logger = render_document_chunk.get_logger()
    logger.info(u'Starting to render document chunk: %s' %
                ','.join([str(pk) for pk in pks]))
    base_url = base_url or settings.SITE_URL
    for pk in pks:
        # calling the task without delay here since we want to localize
        # the processing of the chunk in one process
        result = render_document(pk, cache_control, base_url, force=force)
        if result:
            logger.error(u'Error while rendering document %s with error: %s' %
                         (pk, result))
    logger.info(u'Finished rendering of document chunk')

@task(throws=(StaleDocumentsRenderingInProgress,))
def acquire_render_lock():
    """
    A task to acquire the render document lock
    """
    if render_lock.locked():
        # fail loudly if this is running already
        # may indicate a problem with the schedule of this task
        raise StaleDocumentsRenderingInProgress
    render_lock.acquire()

@task
def release_render_lock():
    """
    A task to release the render document lock
    """
    render_lock.release()

@task
def render_stale_documents(log=None):
    """Simple task wrapper for rendering stale documents"""
    stale_docs = Document.objects.get_by_stale_rendering().distinct()
    stale_docs_count = stale_docs.count()
    if stale_docs_count == 0:
        # not stale documents to render
        return

    if log is None:
        # fetch a logger in case none is given
        log = render_stale_documents.get_logger()

    log.info('Found %s stale documents' % stale_docs_count)
    stale_pks = stale_docs.values_list('pk', flat=True)

    pre_task = acquire_render_lock.si()
    render_tasks = [render_document_chunk.si(pks)
                    for pks in chunked(stale_pks, 5)]
    post_task = release_render_lock.si()

    chord_flow(pre_task, render_tasks, post_task).apply_async()

@task
def build_json_data_for_document(pk, stale):
    """Force-refresh cached JSON data after rendering."""
    document = Document.objects.get(pk=pk)
    document.get_json_data(stale=stale)

    # If we're a translation, rebuild our source doc's JSON so its
    # translation list includes our last edit date.
    if document.parent is not None:
        parent_json = json.dumps(document.parent.build_json_data())
        Document.objects.filter(pk=document.parent.pk).update(json=parent_json)

@task
def move_page(locale, slug, new_slug, email):
    transaction.set_autocommit(False)
    User = get_user_model()
    try:
        user = User.objects.get(email=email)
    except User.DoesNotExist:
        transaction.rollback()
        logging.error('Page move failed: no user with email address %s' %
                      email)
        return
    try:
        doc = Document.objects.get(locale=locale, slug=slug)
    except Document.DoesNotExist:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale
    %(locale)s, but no such document exists.
    """ % {'slug': slug, 'locale': locale}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return
    try:
        doc._move_tree(new_slug, user=user)
    except PageMoveError as e:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale
    %(locale)s, but could not be completed.

    Diagnostic info:

    %(message)s
    """ % {'slug': slug, 'locale': locale, 'message': e.message}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return
    except Exception as e:
        transaction.rollback()
        message = """
    Page move failed.

    Move was requested for document with slug %(slug)s in locale %(locale)s,
    but could not be completed.

    %(info)s
    """ % {'slug': slug, 'locale': locale, 'info': e}
        logging.error(message)
        send_mail('Page move failed', message, settings.DEFAULT_FROM_EMAIL,
                  [user.email])
        transaction.set_autocommit(True)
        return

    transaction.commit()
    transaction.set_autocommit(True)

    # Now that we know the move succeeded, re-render the whole tree.
    for moved_doc in [doc] + doc.get_descendants():
        moved_doc.schedule_rendering('max-age=0')

    subject = 'Page move completed: ' + slug + ' (' + locale + ')'
    full_url = settings.SITE_URL + '/' + locale + '/docs/' + new_slug
    message = """
Page move completed.

The move requested for the document with slug %(slug)s in locale
%(locale)s, and all its children, has been completed.

You can now view this document at its new location: %(full_url)s.
    """ % {'slug': slug, 'locale': locale, 'full_url': full_url}
    send_mail(subject, message, settings.DEFAULT_FROM_EMAIL,
              [user.email])

@task
def update_community_stats():
    cursor = connection.cursor()
    try:
        cursor.execute("""
            SELECT count(creator_id)
            FROM
              (SELECT DISTINCT creator_id
               FROM wiki_revision
               WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
            """)
        contributors = cursor.fetchone()

        cursor.execute("""
            SELECT count(locale)
            FROM
              (SELECT DISTINCT wd.locale
               FROM wiki_document wd,
                                  wiki_revision wr
               WHERE wd.id = wr.document_id
                 AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
            """)
        locales = cursor.fetchone()
    finally:
        cursor.close()

    community_stats = {}

    try:
        community_stats['contributors'] = contributors[0]
        community_stats['locales'] = locales[0]
    except IndexError:
        community_stats = None

    # storing a None value in cache allows a better check for
    # emptiness in the view
    if 0 in community_stats.values():
        community_stats = None

    memcache.set('community_stats', community_stats, 86400)

@task
def delete_old_revision_ips(days=30):
    RevisionIP.objects.delete_old(days=days)

@task
def send_first_edit_email(revision_pk):
    """ Make an 'edited' notification email for first-time editors """
    revision = Revision.objects.get(pk=revision_pk)
    user, doc = revision.creator, revision.document
    subject = (u"[MDN] %(user)s made their first edit, to: %(doc)s" %
               {'user': user.username, 'doc': doc.title})
    message = render_to_string('wiki/email/edited.ltxt',
                               context_dict(revision))
    doc_url = absolutify(doc.get_absolute_url())
    email = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL,
                         to=[config.EMAIL_LIST_FOR_FIRST_EDITS],
                         headers={'X-Kuma-Document-Url': doc_url,
                                  'X-Kuma-Editor-Username': user.username})
    email.send()

class WikiSitemap(GenericSitemap):
    protocol = 'https'
    priority = 0.5

SITEMAP_START = u'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
SITEMAP_ELEMENT = u'<sitemap><loc>%s</loc><lastmod>%s</lastmod></sitemap>'
SITEMAP_END = u'</sitemapindex>'

@task
def build_locale_sitemap(locale):
    """
    For the given locale build the appropriate sitemap file and
    returns the locale, the file names written and timestamp of the
    build.
    """
    now = datetime.utcnow()
    timestamp = '%s+00:00' % now.replace(microsecond=0).isoformat()

    directory = os.path.join(settings.MEDIA_ROOT, 'sitemaps', locale)
    if not os.path.isdir(directory):
        os.makedirs(directory)

    queryset = Document.objects.filter_for_list(locale=locale)
    if queryset.exists():
        names = []
        info = {
            'queryset': queryset,
            'date_field': 'modified',
        }
        sitemap = WikiSitemap(info)
        for page in range(1, sitemap.paginator.num_pages + 1):
            urls = sitemap.get_urls(page=page)
            if page == 1:
                name = 'sitemap.xml'
            else:
                name = 'sitemap_%s.xml' % page
            names.append(name)

            rendered = smart_str(render_to_string('wiki/sitemap.xml',
                                                  {'urls': urls}))
            path = os.path.join(directory, name)
            with open(path, 'w') as sitemap_file:
                sitemap_file.write(rendered)

        return locale, names, timestamp

@task
def build_index_sitemap(results):
    """
    A chord callback task that writes a sitemap index file for the
    given results of :func:`~kuma.wiki.tasks.build_locale_sitemap` task.
    """
    sitemap_parts = [SITEMAP_START]

    for result in results:
        # result can be empty if no documents were found
        if result is not None:
            locale, names, timestamp = result
            for name in names:
                sitemap_url = absolutify('/sitemaps/%s/%s' % (locale, name))
                sitemap_parts.append(SITEMAP_ELEMENT % (sitemap_url, timestamp))

    sitemap_parts.append(SITEMAP_END)

    index_path = os.path.join(settings.MEDIA_ROOT, 'sitemap.xml')
    sitemap_tree = etree.fromstringlist(sitemap_parts)
    with open(index_path, 'w') as index_file:
        sitemap_tree.getroottree().write(index_file,
                                         encoding='utf-8',
                                         pretty_print=True)

@task
def build_sitemaps():
    """
    Build and save sitemap files for every MDN language and as a
    callback save the sitempa index file as well.
    """
    tasks = [build_locale_sitemap.si(locale)
             for locale in settings.MDN_LANGUAGES]
    post_task = build_index_sitemap.s()
    chord(header=tasks, body=post_task).apply_async()

@task
def index_documents(ids, index_pk, reraise=False):
    """
    Index a list of documents into the provided index.

    :arg ids: Iterable of `Document` pks to index.
    :arg index_pk: The `Index` pk of the index to index into.
    :arg reraise: False if you want errors to be swallowed and True
        if you want errors to be thrown.

    .. Note::

       This indexes all the documents in the chunk in one single bulk
       indexing call. Keep that in mind when you break your indexing
       task into chunks.

    """
    from kuma.wiki.models import Document

    cls = WikiDocumentType
    es = cls.get_connection('indexing')
    index = Index.objects.get(pk=index_pk)

    objects = Document.objects.filter(id__in=ids)
    documents = []
    for obj in objects:
        try:
            documents.append(cls.from_django(obj))
        except Exception:
            log.exception('Unable to extract/index document (id: %d)', obj.id)
            if reraise:
                raise

    if documents:
        cls.bulk_index(documents, id_field='id', es=es,
                       index=index.prefixed_name)

@task
def unindex_documents(ids, index_pk):
    """
    Delete a list of documents from the provided index.

    :arg ids: Iterable of `Document` pks to remove.
    :arg index_pk: The `Index` pk of the index to remove items from.
    """
    cls = WikiDocumentType
    es = cls.get_connection('indexing')
    index = Index.objects.get(pk=index_pk)

    cls.bulk_delete(ids, es=es, index=index.prefixed_name)

@task(rate_limit='120/m')
def tidy_revision_content(pk):
    """
    Run tidy over the given revision's content and save it to the
    tidy_content field if the content is not equal to the current value.

    :arg pk: Primary key of `Revision` whose content needs tidying.
    """
    try:
        revision = Revision.objects.get(pk=pk)
    except Revision.DoesNotExist as exc:
        # Retry in 2 minutes
        log.error('Tidy was unable to get revision id: %d. Retrying.', pk)
        tidy_revision_content.retry(countdown=60 * 2, max_retries=5, exc=exc)
    else:
        tidied_content, errors = tidy_content(revision.content)
        if tidied_content != revision.tidied_content:
            Revision.objects.filter(pk=pk).update(
                tidied_content=tidied_content
            )
        # return the errors so we can look them up in the Celery task result
        return errors
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import os
import shutil
import util
import string

# Note: when creating templates, be aware that they are evaulated
# twice, in the context of two different dictionaries.
#  o  keys from keywords should have a # single %, as should "args".
#  o  keys passed in through the macros argument will need %% to
#     escape them for delayed evaluation.  This will include at
#     least %%(builddir)s for all, and doInstall will also get
#     %%(destdir)s

class ShellCommand:
    def __init__(self, *args, **keywords):
        # initialize initialize our keywords to the defaults
        self.__dict__.update(self.keywords)
        # check to make sure that we don't get a keyword we don't expect
        for key in keywords.keys():
            if key not in self.keywords.keys():
                raise TypeError, ("%s.__init__() got an unexpected keyword argument "
                                  "'%s'" % (self.__class__.__name__, key))
        # copy the keywords into our dict, overwriting the defaults
        self.__dict__.update(keywords)
        self.args = string.join(args)
        # pre-fill in the preMake and arguments
        self.command = self.template % self.__dict__

    def execute(self, command):
        print '+', command
        rc = os.system(command)
        if rc:
            raise RuntimeError, ('Shell command "%s" returned '
                                 'non-zero status %d' % (command, rc))

class Automake(ShellCommand):
    # note: no use of %(args)s -- which command would it apply to?
    template = ('cd %%(builddir)s; '
                'aclocal %%(m4DirArgs)s %(acLocalArgs)s; '
		'%(preAutoconf)s autoconf %(autoConfArgs)s; '
		'automake %(autoMakeArgs)s')
    keywords = {'autoConfArgs': '--force',
                'autoMakeArgs': '--copy --force',
		'acLocalArgs': '',
		'preAutoconf': '',
                'm4Dir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.m4Dir:
	    macros.update({'m4DirArgs': '-I %s' %(self.m4Dir)})
        self.execute(self.command %macros)

class Configure(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
		'%(preConfigure)s %%(configure)s '
		'  --prefix=%%(prefix)s '
                '  --sysconfdir=%%(sysconfdir)s '
		'  --datadir=%%(datadir)s '
		'  --mandir=%%(mandir)s --infodir=%%(infodir)s '
		'  %(args)s')
    keywords = {'preConfigure': '',
                'objDir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.objDir:
            macros['mkObjdir'] = 'mkdir -p %s; cd %s;' \
	                         %(self.objDir, self.objDir)
	    macros['configure'] = '../configure'
        else:
            macros['configure'] = './configure'
        self.execute(self.command %macros)

class ManualConfigure(Configure):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
	        '%(preConfigure)s %%(configure)s %(args)s')

class Make(ShellCommand):
    template = 'cd %%(builddir)s; %(preMake)s make %(args)s'
    keywords = {'preMake': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        self.execute(self.command %macros)

class MakeInstall(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%(preMake)s make %(rootVar)s=%%(destdir)s install %(args)s')
    keywords = {'rootVar': 'DESTDIR',
                'preMake': ''}

    def doInstall(self, macros):
	self.execute(self.command %macros)

class InstallFile:

    # make sure that the decimal value really is unreasonable before
    # adding a new translation to this file.
    permmap = {
	1755: 01755,
	4755: 04755,
	755: 0755,
	750: 0750,
	644: 0644,
	640: 0640,
    }

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))

	for fromFile in self.fromFiles:
	    thisdest = dest
	    if dest[-1:] == '/':
		thisdest = dest + fromFile
	    shutil.copyfile(fromFile, thisdest)
	    os.chmod(thisdest, self.mode)

    def __init__(self, fromFiles, toFile, perms = 0644):
	self.toFile = toFile
	if type(fromFiles) is str:
	    self.fromFiles = (fromFiles,)
	else:
	    self.fromFiles = fromFiles
	# notice obviously broken permissions
	if self.permmap.has_key(perms):
	    print 'odd permission %o, correcting to %o: add initial "0"?' \
	          %(perms, permmap[perms])
	    perms = permmap[perms]
	self.mode = perms

class InstallSymlink:

    def doInstall(self, macros):
	os.symlink(self.fromFile, macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile

class RemoveFiles:

    def doInstall(self, macros):
	if self.recursive:
	    os.system("rm -rf %s/%s" %(macros['destdir'], self.filespec))
	else:
	    os.system("rm -f %s/%s" %(macros['destdir'], self.filespec))

    def __init__(self, filespec, recursive=0):
	self.filespec = filespec
	self.recursive = recursive
<commit_msg>use another macro -- still need to do the rest...<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import os
import shutil
import util
import string

# Note: when creating templates, be aware that they are evaulated
# twice, in the context of two different dictionaries.
#  o  keys from keywords should have a # single %, as should "args".
#  o  keys passed in through the macros argument will need %% to
#     escape them for delayed evaluation.  This will include at
#     least %%(builddir)s for all, and doInstall will also get
#     %%(destdir)s

class ShellCommand:
    def __init__(self, *args, **keywords):
        # initialize initialize our keywords to the defaults
        self.__dict__.update(self.keywords)
        # check to make sure that we don't get a keyword we don't expect
        for key in keywords.keys():
            if key not in self.keywords.keys():
                raise TypeError, ("%s.__init__() got an unexpected keyword argument "
                                  "'%s'" % (self.__class__.__name__, key))
        # copy the keywords into our dict, overwriting the defaults
        self.__dict__.update(keywords)
        self.args = string.join(args)
        # pre-fill in the preMake and arguments
        self.command = self.template % self.__dict__

    def execute(self, command):
        print '+', command
        rc = os.system(command)
        if rc:
            raise RuntimeError, ('Shell command "%s" returned '
                                 'non-zero status %d' % (command, rc))

class Automake(ShellCommand):
    # note: no use of %(args)s -- which command would it apply to?
    template = ('cd %%(builddir)s; '
                'aclocal %%(m4DirArgs)s %(acLocalArgs)s; '
		'%(preAutoconf)s autoconf %(autoConfArgs)s; '
		'automake %(autoMakeArgs)s')
    keywords = {'autoConfArgs': '--force',
                'autoMakeArgs': '--copy --force',
		'acLocalArgs': '',
		'preAutoconf': '',
                'm4Dir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.m4Dir:
	    macros.update({'m4DirArgs': '-I %s' %(self.m4Dir)})
        self.execute(self.command %macros)

class Configure(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
		'%(preConfigure)s %%(configure)s '
		'  --prefix=%%(prefix)s '
		'  --bindir=%%(bindir)s '
                '  --sysconfdir=%%(sysconfdir)s '
		'  --datadir=%%(datadir)s '
		'  --mandir=%%(mandir)s --infodir=%%(infodir)s '
		'  %(args)s')
    keywords = {'preConfigure': '',
                'objDir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.objDir:
            macros['mkObjdir'] = 'mkdir -p %s; cd %s;' \
	                         %(self.objDir, self.objDir)
	    macros['configure'] = '../configure'
        else:
            macros['configure'] = './configure'
        self.execute(self.command %macros)

class ManualConfigure(Configure):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
	        '%(preConfigure)s %%(configure)s %(args)s')

class Make(ShellCommand):
    template = 'cd %%(builddir)s; %(preMake)s make %(args)s'
    keywords = {'preMake': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        self.execute(self.command %macros)

class MakeInstall(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%(preMake)s make %(rootVar)s=%%(destdir)s install %(args)s')
    keywords = {'rootVar': 'DESTDIR',
                'preMake': ''}

    def doInstall(self, macros):
	self.execute(self.command %macros)

class InstallFile:

    # make sure that the decimal value really is unreasonable before
    # adding a new translation to this file.
    permmap = {
	1755: 01755,
	4755: 04755,
	755: 0755,
	750: 0750,
	644: 0644,
	640: 0640,
    }

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))

	for fromFile in self.fromFiles:
	    thisdest = dest
	    if dest[-1:] == '/':
		thisdest = dest + fromFile
	    shutil.copyfile(fromFile, thisdest)
	    os.chmod(thisdest, self.mode)

    def __init__(self, fromFiles, toFile, perms = 0644):
	self.toFile = toFile
	if type(fromFiles) is str:
	    self.fromFiles = (fromFiles,)
	else:
	    self.fromFiles = fromFiles
	# notice obviously broken permissions
	if self.permmap.has_key(perms):
	    print 'odd permission %o, correcting to %o: add initial "0"?' \
	          %(perms, permmap[perms])
	    perms = permmap[perms]
	self.mode = perms

class InstallSymlink:

    def doInstall(self, macros):
	os.symlink(self.fromFile, macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile

class RemoveFiles:

    def doInstall(self, macros):
	if self.recursive:
	    os.system("rm -rf %s/%s" %(macros['destdir'], self.filespec))
	else:
	    os.system("rm -f %s/%s" %(macros['destdir'], self.filespec))

    def __init__(self, filespec, recursive=0):
	self.filespec = filespec
	self.recursive = recursive
<|endoftext|>


<commit_before>#! python 3

# a script to compare book prices between Flipkart.com and Amazon.in

#imports
import bs4,re,sys,requests
from math import floor

#RegEX objects
numberRegEx = re.compile(r'(\d)+')

#setting my user-agent
headers = {
    'User-Agent': 'Argus v0.1'
}

#functions for finding isbn
def amazonISBN(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    isbnEle = soup.select('td.bucket > div > ul > li:nth-of-type(4)')
    return (isbnEle[0].text[8:].strip())

def flipkartISBN(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    isbnEle = soup.select('table.specTable > tr:nth-of-type(3) > td:nth-of-type(2)')
    return (isbnEle[0].text.strip())

#functions for finding prices from each website
def amazonPrice(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    priceEle = soup.select('.inlineBlock-display')
    return (priceEle[0].text.strip())

def flipkartPrice(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    priceEle = soup.select('.pricing > div > div > span:nth-of-type(1)')
    deliverEle = soup.select('.default-shipping-charge')
    deliveryCharge = numberRegEx.search(deliverEle[0].contents[0])
    if deliveryCharge != None:
        return ([priceEle[0].text[4:].strip(),deliveryCharge.group(0)])
    else:
        return ([priceEle[0].text[4:].strip(),0])

#functions for returing the correct link from isbn
def amazonLink(isbn):
    return('http://www.amazon.in/gp/product/'+str(isbn))
def flipkartLink(isbn):
    return('http://www.flipkart.com/search?q='+str(isbn))

#driver logic
link1=''
link2=''
if len(sys.argv) == 1:
    print ('Enter the link (Flipkart/Amazon): ')
    myLink = input().upper()
    if 'FLIPKART.COM' in myLink:
        link1 += myLink.lower()
        isbnfromlink = flipkartISBN(link1)
        link2 += amazonLink(isbnfromlink)
    elif 'AMAZON.IN' in myLink:
        link2 += myLink.lower()
        isbnfromlink = amazonISBN(link2)
        link1 += flipkartLink(isbnfromlink)
    else:
        print('\nWrong Link. Only Flipkart.com and Amazon.in Links accepted.')

else:
    if 'flipkart.com'.upper() in sys.argv[1].upper():
        link1 += sys.argv[1]
        isbnfromlink = flipkartISBN(link1)
        link2 += amazonLink(isbnfromlink)
    elif 'amazon.in'.upper() in sys.argv[1].upper():
        link2 += sys.argv[1]
        isbnfromlink = amazonISBN(link2)
        link1 += flipkartLink(isbnfromlink)
    else:
        print('\nWrong Link. Only Flipkart.com and Amazon.in Links accepted.')
try:
    amazonPrice = float(amazonPrice(link2))
    fkprice = flipkartPrice(link1)
    flipkartPrice = float(fkprice[0])+float(fkprice[1])
    print('\n\nAmazon Price: Rs. '+str(amazonPrice)+'\nFlipkart Price: Rs. '+str(flipkartPrice) + ' (includes shipping charges of Rs. '+ str(fkprice[1])+')')
    if amazonPrice < flipkartPrice:
        print('\nAmazon seems to be giving you a better deal. Go to:'+link2)
        price_diff = (flipkartPrice - amazonPrice)
        perc = (price_diff/amazonPrice) * 100
        print('\nThe price differnce is Rs.'+str(price_diff))
        print('\nYou can save '+str(floor(perc))+'% if you order from Amazon.in')
    else:
        print('\nFlipkart seems to be giving you a better deal. Go to:'+link1)
        price_diff = (amazonPrice - flipkartPrice)
        perc = (price_diff/flipkartPrice) * 100
        print('\nThe price differnce is Rs.'+str(price_diff))
        print('\nYou can save '+str(floor(perc))+'% if you order from Flipkart')
except:
    print('\n\n***Error While Parsing. Quitting.***')

#Test Code

#testLinkAMAZON = 'http://www.amazon.in/To-Kill-Mockingbird-Harper-Lee/dp/0099549484'
#testLinkFK = 'http://www.flipkart.com/to-kill-a-mockingbird/p/itme9xt9qzedkusb?pid=9780099549482'
#print(amazonISBN(testLinkAMAZON))
#print(flipkartISBN(testLinkFK))
#print(amazonPrice(testLinkAMAZON))
#print(flipkartPrice(testLinkFK))

#TODOs

#check for arguments, should be either a Flipkart Book link or an Amazon Book link

#find the isbn-10 info, store it globally
#flipkart:#fk-mainbody-id > div > div:nth-child(17) > div.gd-col.gu12 > div.productSpecs.specSection > table:nth-child(2) > tbody > tr:nth-child(3) > td.specsValue
#amazon:#detail_bullets_id > table > tbody > tr > td > div > ul > li:nth-child(4)

#search the book on both websites
#flipkart: http://www.flipkart.com/search?q=<isbn-10 code>
#amazon:http://www.amazon.in/gp/product/<isbn-10 code>

#find price info (include delivery charges if present, //try for fk app price too)
#flipkart:#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.left-section-wrap.size2of5.unit > div > div.price-wrap > div > div.prices > div > span.selling-price.omniture-field
#fk (delivery):#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.left-section-wrap.size2of5.unit > div > div.price-wrap > div > div.default-shipping-charge
#fk (app):#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.right-section-wrap.size3of5.unit > div > div:nth-child(2) > div > div.offers-header.line > span
#amazon:#soldByThirdParty > span
#amazon (delivery):#buyNewInner > div.a-section.a-spacing-small.a-spacing-top-micro > div > span.a-color-base.buyboxShippingLabel > a

#display price and savings from original link
<commit_msg>Added Goodreads rating<commit_after>#! python 3

# a script to compare book prices between Flipkart.com and Amazon.in and display book rating from Goodreads.com
# requires bs4 pip install beautifulsoup4

#imports
import bs4,re,sys,requests
from math import floor

#RegEX objects
numberRegEx = re.compile(r'(\d)+')

#setting my user-agent
headers = {
    'User-Agent': 'Argus v0.1'
}

#function for finding goodreads rating
def goodreadsRating(isbn):
    link='http://www.goodreads.com/search?utf8=%E2%9C%93&query='+isbn
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    ratingEle = soup.select('#bookMeta > span.value.rating > span.average')
    return(ratingEle[0].text)

#functions for finding isbn
def amazonISBN(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    isbnEle = soup.select('td.bucket > div > ul > li:nth-of-type(4)')
    return (isbnEle[0].text[8:].strip())

def flipkartISBN(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    isbnEle = soup.select('table.specTable > tr:nth-of-type(3) > td:nth-of-type(2)')
    return (isbnEle[0].text.strip())

#functions for finding prices from each website
def amazonPrice(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    priceEle = soup.select('.inlineBlock-display')
    return (priceEle[0].text.strip())

def flipkartPrice(link):
    req = requests.get(link, headers = headers)
    req.raise_for_status()
    soup = bs4.BeautifulSoup(req.text, 'html.parser')
    priceEle = soup.select('.pricing > div > div > span:nth-of-type(1)')
    deliverEle = soup.select('.default-shipping-charge')
    deliveryCharge = numberRegEx.search(deliverEle[0].contents[0])
    if deliveryCharge != None:
        return ([priceEle[0].text[4:].strip(),deliveryCharge.group(0)])
    else:
        return ([priceEle[0].text[4:].strip(),0])

#functions for returing the correct link from isbn
def amazonLink(isbn):
    return('http://www.amazon.in/gp/product/'+str(isbn))
def flipkartLink(isbn):
    return('http://www.flipkart.com/search?q='+str(isbn))

#driver logic
link1=''
link2=''
isbnfromlink=''
if len(sys.argv) == 1:
    print ('Enter the link (Flipkart/Amazon): ')
    myLink = input().upper()
    if 'FLIPKART.COM' in myLink:
        link1 += myLink.lower()
        isbnfromlink = flipkartISBN(link1)
        link2 += amazonLink(isbnfromlink)
    elif 'AMAZON.IN' in myLink:
        link2 += myLink.lower()
        isbnfromlink = amazonISBN(link2)
        link1 += flipkartLink(isbnfromlink)
    else:
        print('\nWrong Link. Only Flipkart.com and Amazon.in Links accepted.')

else:
    if 'flipkart.com'.upper() in sys.argv[1].upper():
        link1 += sys.argv[1]
        isbnfromlink = flipkartISBN(link1)
        link2 += amazonLink(isbnfromlink)
    elif 'amazon.in'.upper() in sys.argv[1].upper():
        link2 += sys.argv[1]
        isbnfromlink = amazonISBN(link2)
        link1 += flipkartLink(isbnfromlink)
    else:
        print('\nWrong Link. Only Flipkart.com and Amazon.in Links accepted.')
try:
    amazonPrice = float(amazonPrice(link2))
    fkprice = flipkartPrice(link1)
    flipkartPrice = float(fkprice[0])+float(fkprice[1])
    print('\n\nAmazon Price: Rs. '+str(amazonPrice)+'\nFlipkart Price: Rs. '+str(flipkartPrice) + ' (includes shipping charges of Rs. '+ str(fkprice[1])+')')
    if amazonPrice < flipkartPrice:
        print('\nAmazon seems to be giving you a better deal. Go to:'+link2)
        price_diff = (flipkartPrice - amazonPrice)
        perc = (price_diff/amazonPrice) * 100
        print('\nThe price differnce is Rs.'+str(price_diff))
        print('\nYou can save '+str(floor(perc))+'% if you order from Amazon.in')
        print('Getting Goodreads rating. Wait..')
        print('\nRating of Book on Goodreads: '+goodreadsRating(isbnfromlink))
    else:
        print('\nFlipkart seems to be giving you a better deal. Go to:'+link1)
        price_diff = (amazonPrice - flipkartPrice)
        perc = (price_diff/flipkartPrice) * 100
        print('\nThe price differnce is Rs.'+str(price_diff))
        print('\nYou can save '+str(floor(perc))+'% if you order from Flipkart')
        print('Getting Goodreads rating. Wait..')
        print('\nRating of Book on Goodreads: '+roodreadsRating(isbnfromlink))
except:
    print('\n\n***Error While Parsing. Quitting.***')

#Test Code

#testLinkAMAZON = 'http://www.amazon.in/To-Kill-Mockingbird-Harper-Lee/dp/0099549484'
#testLinkFK = 'http://www.flipkart.com/to-kill-a-mockingbird/p/itme9xt9qzedkusb?pid=9780099549482'
#print(amazonISBN(testLinkAMAZON))
#print(flipkartISBN(testLinkFK))
#print(amazonPrice(testLinkAMAZON))
#print(flipkartPrice(testLinkFK))

#TODOs

#check for arguments, should be either a Flipkart Book link or an Amazon Book link

#find the isbn-10 info, store it globally
#flipkart:#fk-mainbody-id > div > div:nth-child(17) > div.gd-col.gu12 > div.productSpecs.specSection > table:nth-child(2) > tbody > tr:nth-child(3) > td.specsValue
#amazon:#detail_bullets_id > table > tbody > tr > td > div > ul > li:nth-child(4)

#search the book on both websites
#flipkart: http://www.flipkart.com/search?q=<isbn-10 code>
#amazon:http://www.amazon.in/gp/product/<isbn-10 code>

#find price info (include delivery charges if present, //try for fk app price too)
#flipkart:#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.left-section-wrap.size2of5.unit > div > div.price-wrap > div > div.prices > div > span.selling-price.omniture-field
#fk (delivery):#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.left-section-wrap.size2of5.unit > div > div.price-wrap > div > div.default-shipping-charge
#fk (app):#fk-mainbody-id > div > div:nth-child(7) > div > div.right-col-wrap.lastUnit > div > div > div.shop-section-wrap > div > div.section-wrap.line.section > div.right-section-wrap.size3of5.unit > div > div:nth-child(2) > div > div.offers-header.line > span
#amazon:#soldByThirdParty > span
#amazon (delivery):#buyNewInner > div.a-section.a-spacing-small.a-spacing-top-micro > div > span.a-color-base.buyboxShippingLabel > a

#display price and savings from original link
<|endoftext|>