<commit_before># Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

EXCLUDED_PATHS = (
    r"breakpad[\\\/].*",
    r"skia[\\\/].*",
    r"v8[\\\/].*",
)

def CheckChangeOnUpload(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  # TODO(maruel): Run it in a separate thread to parallelize checks?
  results.extend(CheckTreeIsOpen(
      input_api,
      output_api,
      'http://chromium-status.appspot.com/status',
      '0',
      'http://chromium-status.appspot.com/current?format=raw'))
  results.extend(CheckTryJobExecution(input_api, output_api))
  # These builders are just too slow.
  IGNORED_BUILDERS = [
    'Chromium XP',
    'Chromium Mac',
    'Chromium Mac (valgrind)',
    'Chromium Mac UI (valgrind)(1)',
    'Chromium Mac UI (valgrind)(2)',
    'Chromium Mac UI (valgrind)(3)',
    'Chromium Mac (tsan)',
    'Webkit Mac (valgrind)',
    'Chromium Linux',
    'Chromium Linux x64',
    'Linux Tests (valgrind)(1)',
    'Linux Tests (valgrind)(2)',
    'Linux Tests (valgrind)(3)',
    'Linux Tests (valgrind)(4)',
    'Webkit Linux (valgrind layout)',
  ]
  results.extend(CheckPendingBuilds(
      input_api,
      output_api,
      'http://build.chromium.org/buildbot/waterfall/json/builders',
      6,
      IGNORED_BUILDERS))
  return results

def CheckTryJobExecution(input_api, output_api):
  outputs = []
  if not input_api.change.issue or not input_api.change.patchset:
    return outputs
  url = "http://codereview.chromium.org/%d/get_build_results/%d" % (
            input_api.change.issue, input_api.change.patchset)
  PLATFORMS = ('win', 'linux', 'mac')
  try:
    connection = input_api.urllib2.urlopen(url)
    # platform|status|url
    values = [item.split('|', 2) for item in connection.read().splitlines()]
    connection.close()
    if not values:
      # It returned an empty list. Probably a private review.
      return outputs
    # Reformat as an dict of platform: [status, url]
    values = dict([[v[0], [v[1], v[2]]] for v in values])
    for platform in PLATFORMS:
      values.setdefault(platform, ['not started', ''])
    message = None
    non_success = [k.upper() for k,v in values.iteritems() if v[0] != 'success']
    if 'failure' in [v[0] for v in values.itervalues()]:
      message = 'Try job failures on %s!\n' % ', '.join(non_success)
    elif non_success:
      message = ('Unfinished (or not even started) try jobs on '
                 '%s.\n') % ', '.join(non_success)
    if message:
      message += (
          'Is try server wrong or broken? Please notify maruel@chromium.org. '
          'Thanks.\n')
      outputs.append(output_api.PresubmitPromptWarning(message=message))
  except input_api.urllib2.HTTPError, e:
    if e.code == 404:
      # Fallback to no try job.
      # TODO(maruel): Change to a PresubmitPromptWarning once the try server is
      # stable enough and it seems to work fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'You should try the patch first.'))
    else:
      # Another HTTP error happened, warn the user.
      # TODO(maruel): Change to a PresubmitPromptWarning once it deemed to work
      # fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'Got %s while looking for try job status.' % str(e)))
  return outputs

def CheckTreeIsOpen(input_api, output_api, url, closed, url_text):
  """Similar to the one in presubmit_canned_checks except it shows an helpful
  status text instead.
  """
  assert(input_api.is_committing)
  try:
    connection = input_api.urllib2.urlopen(url)
    status = connection.read()
    connection.close()
    if input_api.re.match(closed, status):
      long_text = status + '\n' + url
      try:
        connection = input_api.urllib2.urlopen(url_text)
        long_text = connection.read().strip()
        connection.close()
      except IOError:
        pass
      return [output_api.PresubmitError("The tree is closed.",
                                        long_text=long_text)]
  except IOError:
    pass
  return []

def CheckPendingBuilds(input_api, output_api, url, max_pendings, ignored):
  try:
    connection = input_api.urllib2.urlopen(url)
    raw_data = connection.read()
    connection.close()
    try:
      import simplejson
      data = simplejson.loads(raw_data)
    except ImportError:
      # simplejson is much safer. But we should be just fine enough with that:
      data = eval(raw_data.replace('null', 'None'))
    out = []
    for (builder_name, builder)  in data.iteritems():
      if builder_name in ignored:
        continue
      pending_builds_len = len(builder.get('pending_builds', []))
      if pending_builds_len > max_pendings:
        out.append('%s has %d build(s) pending' %
                   (builder_name, pending_builds_len))
    if out:
      return [output_api.PresubmitPromptWarning(
          'Build(s) pending. It is suggested to wait that no more than %d '
              'builds are pending.' % max_pendings,
          long_text='\n'.join(out))]
  except IOError:
    # Silently pass.
    pass
  return []

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<commit_msg>Use the newly added CheckLicense() canned check to verify license boiler plate on source files.<commit_after># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

EXCLUDED_PATHS = (
    r"breakpad[\\\/].*",
    r"skia[\\\/].*",
    r"v8[\\\/].*",
)

_LICENSE_HEADER = (
     r".*? Copyright \(c\) 20\d\d The Chromium Authors\. All rights reserved\."
       "\n"
     r".*? Use of this source code is governed by a BSD-style license that can "
       "be\n"
     r".*? found in the LICENSE file\."
       "\n"
)

def _CommonChecks(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckLicense(
      input_api, output_api, _LICENSE_HEADER, sources))
  return results

def CheckChangeOnUpload(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  # TODO(maruel): Run it in a separate thread to parallelize checks?
  results.extend(CheckTreeIsOpen(
      input_api,
      output_api,
      'http://chromium-status.appspot.com/status',
      '0',
      'http://chromium-status.appspot.com/current?format=raw'))
  results.extend(CheckTryJobExecution(input_api, output_api))
  # These builders are just too slow.
  IGNORED_BUILDERS = [
    'Chromium XP',
    'Chromium Mac',
    'Chromium Mac (valgrind)',
    'Chromium Mac UI (valgrind)(1)',
    'Chromium Mac UI (valgrind)(2)',
    'Chromium Mac UI (valgrind)(3)',
    'Chromium Mac (tsan)',
    'Webkit Mac (valgrind)',
    'Chromium Linux',
    'Chromium Linux x64',
    'Linux Tests (valgrind)(1)',
    'Linux Tests (valgrind)(2)',
    'Linux Tests (valgrind)(3)',
    'Linux Tests (valgrind)(4)',
    'Webkit Linux (valgrind layout)',
  ]
  results.extend(CheckPendingBuilds(
      input_api,
      output_api,
      'http://build.chromium.org/buildbot/waterfall/json/builders',
      6,
      IGNORED_BUILDERS))
  return results

def CheckTryJobExecution(input_api, output_api):
  outputs = []
  if not input_api.change.issue or not input_api.change.patchset:
    return outputs
  url = "http://codereview.chromium.org/%d/get_build_results/%d" % (
            input_api.change.issue, input_api.change.patchset)
  PLATFORMS = ('win', 'linux', 'mac')
  try:
    connection = input_api.urllib2.urlopen(url)
    # platform|status|url
    values = [item.split('|', 2) for item in connection.read().splitlines()]
    connection.close()
    if not values:
      # It returned an empty list. Probably a private review.
      return outputs
    # Reformat as an dict of platform: [status, url]
    values = dict([[v[0], [v[1], v[2]]] for v in values])
    for platform in PLATFORMS:
      values.setdefault(platform, ['not started', ''])
    message = None
    non_success = [k.upper() for k,v in values.iteritems() if v[0] != 'success']
    if 'failure' in [v[0] for v in values.itervalues()]:
      message = 'Try job failures on %s!\n' % ', '.join(non_success)
    elif non_success:
      message = ('Unfinished (or not even started) try jobs on '
                 '%s.\n') % ', '.join(non_success)
    if message:
      message += (
          'Is try server wrong or broken? Please notify maruel@chromium.org. '
          'Thanks.\n')
      outputs.append(output_api.PresubmitPromptWarning(message=message))
  except input_api.urllib2.HTTPError, e:
    if e.code == 404:
      # Fallback to no try job.
      # TODO(maruel): Change to a PresubmitPromptWarning once the try server is
      # stable enough and it seems to work fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'You should try the patch first.'))
    else:
      # Another HTTP error happened, warn the user.
      # TODO(maruel): Change to a PresubmitPromptWarning once it deemed to work
      # fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'Got %s while looking for try job status.' % str(e)))
  return outputs

def CheckTreeIsOpen(input_api, output_api, url, closed, url_text):
  """Similar to the one in presubmit_canned_checks except it shows an helpful
  status text instead.
  """
  assert(input_api.is_committing)
  try:
    connection = input_api.urllib2.urlopen(url)
    status = connection.read()
    connection.close()
    if input_api.re.match(closed, status):
      long_text = status + '\n' + url
      try:
        connection = input_api.urllib2.urlopen(url_text)
        long_text = connection.read().strip()
        connection.close()
      except IOError:
        pass
      return [output_api.PresubmitError("The tree is closed.",
                                        long_text=long_text)]
  except IOError:
    pass
  return []

def CheckPendingBuilds(input_api, output_api, url, max_pendings, ignored):
  try:
    connection = input_api.urllib2.urlopen(url)
    raw_data = connection.read()
    connection.close()
    try:
      import simplejson
      data = simplejson.loads(raw_data)
    except ImportError:
      # simplejson is much safer. But we should be just fine enough with that:
      data = eval(raw_data.replace('null', 'None'))
    out = []
    for (builder_name, builder)  in data.iteritems():
      if builder_name in ignored:
        continue
      pending_builds_len = len(builder.get('pending_builds', []))
      if pending_builds_len > max_pendings:
        out.append('%s has %d build(s) pending' %
                   (builder_name, pending_builds_len))
    if out:
      return [output_api.PresubmitPromptWarning(
          'Build(s) pending. It is suggested to wait that no more than %d '
              'builds are pending.' % max_pendings,
          long_text='\n'.join(out))]
  except IOError:
    # Silently pass.
    pass
  return []

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
details on the presubmit API built into gcl.
"""

# Files with these extensions will be considered source files.
SOURCE_FILE_EXTENSIONS = [
    '.c', '.cc', '.cpp', '.h', '.m', '.mm', '.py', '.mk', '.am', '.json',
]
EXCLUDED_PATHS = [
    r"breakpad[\\\/].*",
    r"chrome[\\\/]Debug[\\\/].*",
    r"chrome[\\\/]Release[\\\/].*",
    r"sconsbuild[\\\/].*",
    r"xcodebuild[\\\/].*",
    r"skia[\\\/].*",
    r".*third_party[\\\/].*",
    r"v8[\\\/].*",
]

def ReadFile(path):
  """Given a path, returns the full contents of the file.

  Reads files in binary format.
  """
  fo = open(path, 'rb')
  try:
    contents = fo.read()
  finally:
    fo.close()
  return contents

def CheckChangeOnUpload(input_api, output_api):
  # TODO(maruel): max_cols is temporarily disabled. Reenable once the source
  # tree is in better shape.
  return LocalChecks(input_api, output_api, max_cols=0)

def CheckChangeOnCommit(input_api, output_api):
  # TODO(maruel): max_cols is temporarily disabled. Reenable once the source
  # tree is in better shape.
  return (LocalChecks(input_api, output_api, max_cols=0) +
          input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))

def LocalChecks(input_api, output_api, max_cols=80):
  """Reports an error if for any source file in SOURCE_FILE_EXTENSIONS:
     - uses CR (or CRLF)
     - contains a TAB
     - has a line that ends with whitespace
     - contains a line >|max_cols| cols unless |max_cols| is 0.
     - File does not end in a newline, or ends in more than one.

  Note that the whole file is checked, not only the changes.
  """
  C_SOURCE_FILE_EXTENSIONS = ('.c', '.cc', '.cpp', '.h', '.inl')
  cr_files = []
  eof_files = []
  results = []
  excluded_paths = [input_api.re.compile(x) for x in EXCLUDED_PATHS]
  files = input_api.AffectedFiles(include_deletes=False)
  for f in files:
    path = f.LocalPath()
    root, ext = input_api.os_path.splitext(path)
    # Look for unsupported extensions.
    if not ext in SOURCE_FILE_EXTENSIONS:
      continue
    # Look for excluded paths.
    found = False
    for item in excluded_paths:
      if item.match(path):
        found = True
        break
    if found:
      continue

    # Need to read the file ourselves since AffectedFile.NewContents()
    # will normalize line endings.
    contents = ReadFile(path)
    if '\r' in contents:
      cr_files.append(path)

    # Check that the file ends in one and only one newline character.
    if len(contents) > 0 and (contents[-1:] != "\n" or contents[-2:-1] == "\n"):
      eof_files.append(path)

    local_errors = []
    # Remove end of line character.
    lines = contents.splitlines()
    line_num = 1
    for line in lines:
      if line.endswith(' '):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s ends with whitespaces.' %
            (path, line_num)))
      # Accept lines with http://, https:// and C #define/#pragma/#include to
      # exceed the max_cols rule.
      if (max_cols and
          len(line) > max_cols and
          not 'http://' in line and
          not 'https://' in line and
          not (line[0] == '#' and ext in C_SOURCE_FILE_EXTENSIONS)):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s has %s chars, please reduce to %d chars.' %
            (path, line_num, len(line), max_cols)))
      if '\t' in line:
        local_errors.append(output_api.PresubmitError(
            "%s, line %s contains a tab character." %
            (path, line_num)))
      line_num += 1
      # Just show the first 5 errors.
      if len(local_errors) == 6:
        local_errors.pop()
        local_errors.append(output_api.PresubmitError("... and more."))
        break
    results.extend(local_errors)

  if cr_files:
    results.append(output_api.PresubmitError(
        'Found CR (or CRLF) line ending in these files, please use only LF:',
        items=cr_files))
  if eof_files:
    results.append(output_api.PresubmitError(
        'These files should end in one (and only one) newline character:',
        items=eof_files))
  return results
<commit_msg>Add presubmit checks to look for BUG= and TEST= tags in change list description (Part 2 of 2).<commit_after>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
details on the presubmit API built into gcl.
"""

# Files with these extensions will be considered source files.
SOURCE_FILE_EXTENSIONS = [
    '.c', '.cc', '.cpp', '.h', '.m', '.mm', '.py', '.mk', '.am', '.json',
]
EXCLUDED_PATHS = [
    r"breakpad[\\\/].*",
    r"chrome[\\\/]Debug[\\\/].*",
    r"chrome[\\\/]Release[\\\/].*",
    r"sconsbuild[\\\/].*",
    r"xcodebuild[\\\/].*",
    r"skia[\\\/].*",
    r".*third_party[\\\/].*",
    r"v8[\\\/].*",
]

def ReadFile(path):
  """Given a path, returns the full contents of the file.

  Reads files in binary format.
  """
  fo = open(path, 'rb')
  try:
    contents = fo.read()
  finally:
    fo.close()
  return contents

def CheckChangeOnUpload(input_api, output_api):
  # TODO(maruel): max_cols is temporarily disabled. Reenable once the source
  # tree is in better shape.
  return (LocalChecks(input_api, output_api, max_cols=0) +
      input_api.canned_checks.CheckChangeHasBugField(input_api, output_api) +
      input_api.canned_checks.CheckChangeHasTestField(input_api, output_api))

def CheckChangeOnCommit(input_api, output_api):
  # TODO(maruel): max_cols is temporarily disabled. Reenable once the source
  # tree is in better shape.
  return (LocalChecks(input_api, output_api, max_cols=0) +
      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api) +
      input_api.canned_checks.CheckChangeHasBugField(input_api, output_api) +
      input_api.canned_checks.CheckChangeHasTestField(input_api, output_api))

def LocalChecks(input_api, output_api, max_cols=80):
  """Reports an error if for any source file in SOURCE_FILE_EXTENSIONS:
     - uses CR (or CRLF)
     - contains a TAB
     - has a line that ends with whitespace
     - contains a line >|max_cols| cols unless |max_cols| is 0.
     - File does not end in a newline, or ends in more than one.

  Note that the whole file is checked, not only the changes.
  """
  C_SOURCE_FILE_EXTENSIONS = ('.c', '.cc', '.cpp', '.h', '.inl')
  cr_files = []
  eof_files = []
  results = []
  excluded_paths = [input_api.re.compile(x) for x in EXCLUDED_PATHS]
  files = input_api.AffectedFiles(include_deletes=False)
  for f in files:
    path = f.LocalPath()
    root, ext = input_api.os_path.splitext(path)
    # Look for unsupported extensions.
    if not ext in SOURCE_FILE_EXTENSIONS:
      continue
    # Look for excluded paths.
    found = False
    for item in excluded_paths:
      if item.match(path):
        found = True
        break
    if found:
      continue

    # Need to read the file ourselves since AffectedFile.NewContents()
    # will normalize line endings.
    contents = ReadFile(path)
    if '\r' in contents:
      cr_files.append(path)

    # Check that the file ends in one and only one newline character.
    if len(contents) > 0 and (contents[-1:] != "\n" or contents[-2:-1] == "\n"):
      eof_files.append(path)

    local_errors = []
    # Remove end of line character.
    lines = contents.splitlines()
    line_num = 1
    for line in lines:
      if line.endswith(' '):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s ends with whitespaces.' %
            (path, line_num)))
      # Accept lines with http://, https:// and C #define/#pragma/#include to
      # exceed the max_cols rule.
      if (max_cols and
          len(line) > max_cols and
          not 'http://' in line and
          not 'https://' in line and
          not (line[0] == '#' and ext in C_SOURCE_FILE_EXTENSIONS)):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s has %s chars, please reduce to %d chars.' %
            (path, line_num, len(line), max_cols)))
      if '\t' in line:
        local_errors.append(output_api.PresubmitError(
            "%s, line %s contains a tab character." %
            (path, line_num)))
      line_num += 1
      # Just show the first 5 errors.
      if len(local_errors) == 6:
        local_errors.pop()
        local_errors.append(output_api.PresubmitError("... and more."))
        break
    results.extend(local_errors)

  if cr_files:
    results.append(output_api.PresubmitError(
        'Found CR (or CRLF) line ending in these files, please use only LF:',
        items=cr_files))
  if eof_files:
    results.append(output_api.PresubmitError(
        'These files should end in one (and only one) newline character:',
        items=eof_files))
  return results
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2007 rPath, Inc.
#
# All rights reserved
#

import optparse
import os
import simplejson
import sys
import time

from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from mint import buildtypes
from mint import buildtemplates
from mint import data as mintdata

from mcp import client
from mcp import queue

def main(envArgs = sys.argv[1:]):
    usage = "usage: %prog [options] trovespec"
    parser = optparse.OptionParser(usage)

    parser.add_option("-d", "--dir", dest = "directory", default = '.',
                      help = "write output to DIR", metavar = "DIR")

    parser.add_option("-q", "--quiet",
                      action = "store_false", dest = "verbose", default = True,
                      help = "don't print status messages to stdout")

    parser.add_option("-T", "--title", dest = "title", help = "Project Title")

    parser.add_option("-n", "--name", dest = "name", help = "Name of build")

    parser.add_option("-c", "--config", dest = 'config',
                      help = "conary config", action = 'append', default = [])

    parser.add_option("-H", "--host", dest = 'queueHost',
                      help = "Hostname of message queue", default = '127.0.0.1')

    parser.add_option("-P", "--port", dest = 'queuePort', type="int",
                      help = "Port of message queue", default = 61613)

    parser.add_option("-t", "--type", dest = "TYPE",
                     help = "Type of build")

    parser.add_option("-a", "--advanced", dest = "advanced",
                      help = "Build specific avanced options",
                      default = [], action = "append")

    (options, args) = parser.parse_args(envArgs)

    if options.name is None:
        options.name = options.title

    if not len(args) == 1:
        parser.print_help()
        parser.error("trovespec is required")

    missingOptions = [x[0] for x in options.__dict__.iteritems() \
                          if x[1] is None]
    if missingOptions:
        parser.print_help()
        parser.error("These options must be set: %s." % \
                         (', '.join(missingOptions)))

    if options.TYPE not in buildtypes.validBuildTypes:
        parser.print_help()
        parser.error("Type must be one of: %s" % \
                         (', '.join(buildtypes.validBuildTypes)))

    buildData = {}
    buildData['serialVersion'] = 1

    cfg = conarycfg.ConaryConfiguration(True)
    cfg.initializeFlavors()

    cc = conaryclient.ConaryClient(cfg)
    nc = cc.getRepos()

    n, v, f = cmdline.parseTroveSpec(args[0])
    NVF = nc.findTrove(None, (n, v, f), cc.cfg.flavor)[0]

    labelStr = str(NVF[1].branch().label())
    buildData['troveName'] = NVF[0]
    buildData['troveVersion'] = NVF[1].freeze()
    buildData['troveFlavor'] = NVF[2].freeze()

    UUID = labelStr.split('@')[0] + '-build-' + \
        str(sum([(256 ** x[0]) * x[1] for x in \
                     enumerate([ord(x) for x in os.urandom(8)])]))

    buildData['UUID'] = UUID
    buildData['buildType'] = buildtypes.validBuildTypes[options.TYPE]
    buildData['name'] = options.name

    buildData['project'] = {}
    buildData['project']['hostname'] = labelStr.split('.')[0]
    buildData['project']['label'] = labelStr
    buildData['project']['conaryCfg'] = '\n'.join(options.config)
    buildData['project']['name'] = options.title

    queueName = ''.join([hex(ord(x))[2:] for x in os.urandom(16)])
    buildData['outputQueue'] = queueName
    outputQueue = queue.Queue(options.queueHost, options.queuePort,
                              queueName, timeOut = None)

    buildData['type'] = 'build'

    cfg = client.MCPClientConfig()
    cfg.configLine('queueHost %s' % options.queueHost)
    cfg.configLine('queuePort %d' % options.queuePort)
    mcpClient = client.MCPClient(cfg)

    buildData['data'] = {}
    buildData['data']['jsversion'] = str(mcpClient.getJSVersion())
    buildData['data'].update(dict([x.split(' ', 1) for x in options.advanced]))

    template = buildtemplates.getDataTemplate(buildtypes.__dict__[options.TYPE])

    for key, val in [x for x in buildData['data'].iteritems()]:
        dataType = template.get(key, [mintdata.RDT_STRING])[0]
        if dataType == mintdata.RDT_INT:
            buildData['data'][key] = int(val)
        elif dataType == mintdata.RDT_BOOL:
            buildData['data'][key] = bool(int(val))

    timeSubmitted = time.time()
    timeStarted = None
    timeBuilt = None
    timeFinished = None
    print "submitting:", UUID
    mcpClient.submitJob(simplejson.dumps(buildData))

    status, statusMessage = None, None
    while status not in ('built', 'finished', 'failed'):
        try:
            newStatus, newStatusMessage = mcpClient.jobStatus(UUID)
        except Exception, e:
            newStatus, newStatusMessage = 'error', e
        if (newStatus != 'waiting') and (status == 'waiting'):
            timeStarted = time.time()
        if (newStatus != status) or (statusMessage != newStatusMessage):
            status, statusMessage = newStatus, newStatusMessage
            print "%-79s\x0d" % ("%s: %s" % (status, statusMessage)),
            sys.stdout.flush()
        else:
            time.sleep(1)
    print ""

    if status == 'built':
        timeBuilt = time.time()
        dataStr = outputQueue.read()
        data = simplejson.loads(dataStr)
        for url, type in data['urls']:
            os.system('wget %s -P %s' % (url, options.directory))
        mcpClient.stopJob(UUID)

    timeFinished = time.time()
    mcpClient.disconnect()
    outputQueue.disconnect()

    print "Seconds waiting:", timeStarted - timeSubmitted
    if timeBuilt:
        print "Seconds building:", timeBuilt - timeStarted
        print "Seconds downloading:", timeFinished - timeBuilt
    print "Total:", timeFinished - timeStarted

if __name__ == '__main__':
    main()
<commit_msg>fix status usage<commit_after>#!/usr/bin/python
#
# Copyright (c) 2007 rPath, Inc.
#
# All rights reserved
#

import optparse
import os
import simplejson
import sys
import time

from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from mint import buildtypes
from mint import buildtemplates
from mint import data as mintdata

from mcp import client
from mcp import queue
from mcp import jobstatus

def main(envArgs = sys.argv[1:]):
    usage = "usage: %prog [options] trovespec"
    parser = optparse.OptionParser(usage)

    parser.add_option("-d", "--dir", dest = "directory", default = '.',
                      help = "write output to DIR", metavar = "DIR")

    parser.add_option("-q", "--quiet",
                      action = "store_false", dest = "verbose", default = True,
                      help = "don't print status messages to stdout")

    parser.add_option("-T", "--title", dest = "title", help = "Project Title")

    parser.add_option("-n", "--name", dest = "name", help = "Name of build")

    parser.add_option("-c", "--config", dest = 'config',
                      help = "conary config", action = 'append', default = [])

    parser.add_option("-H", "--host", dest = 'queueHost',
                      help = "Hostname of message queue", default = '127.0.0.1')

    parser.add_option("-P", "--port", dest = 'queuePort', type="int",
                      help = "Port of message queue", default = 61613)

    parser.add_option("-t", "--type", dest = "TYPE",
                     help = "Type of build")

    parser.add_option("-a", "--advanced", dest = "advanced",
                      help = "Build specific avanced options",
                      default = [], action = "append")

    (options, args) = parser.parse_args(envArgs)

    if options.name is None:
        options.name = options.title

    if not len(args) == 1:
        parser.print_help()
        parser.error("trovespec is required")

    missingOptions = [x[0] for x in options.__dict__.iteritems() \
                          if x[1] is None]
    if missingOptions:
        parser.print_help()
        parser.error("These options must be set: %s." % \
                         (', '.join(missingOptions)))

    if options.TYPE not in buildtypes.validBuildTypes:
        parser.print_help()
        parser.error("Type must be one of: %s" % \
                         (', '.join(buildtypes.validBuildTypes)))

    buildData = {}
    buildData['serialVersion'] = 1

    cfg = conarycfg.ConaryConfiguration(True)
    cfg.initializeFlavors()

    cc = conaryclient.ConaryClient(cfg)
    nc = cc.getRepos()

    n, v, f = cmdline.parseTroveSpec(args[0])
    NVF = nc.findTrove(None, (n, v, f), cc.cfg.flavor)[0]

    labelStr = str(NVF[1].branch().label())
    buildData['troveName'] = NVF[0]
    buildData['troveVersion'] = NVF[1].freeze()
    buildData['troveFlavor'] = NVF[2].freeze()

    UUID = labelStr.split('@')[0] + '-build-' + \
        str(sum([(256 ** x[0]) * x[1] for x in \
                     enumerate([ord(x) for x in os.urandom(8)])]))

    buildData['UUID'] = UUID
    buildData['buildType'] = buildtypes.validBuildTypes[options.TYPE]
    buildData['name'] = options.name

    buildData['project'] = {}
    buildData['project']['hostname'] = labelStr.split('.')[0]
    buildData['project']['label'] = labelStr
    buildData['project']['conaryCfg'] = '\n'.join(options.config)
    buildData['project']['name'] = options.title

    queueName = ''.join([hex(ord(x))[2:] for x in os.urandom(16)])
    buildData['outputQueue'] = queueName
    outputQueue = queue.Queue(options.queueHost, options.queuePort,
                              queueName, timeOut = None)

    buildData['type'] = 'build'

    cfg = client.MCPClientConfig()
    cfg.configLine('queueHost %s' % options.queueHost)
    cfg.configLine('queuePort %d' % options.queuePort)
    mcpClient = client.MCPClient(cfg)

    buildData['data'] = {}
    buildData['data']['jsversion'] = str(mcpClient.getJSVersion())
    buildData['data'].update(dict([x.split(' ', 1) for x in options.advanced]))

    template = buildtemplates.getDataTemplate(buildtypes.__dict__[options.TYPE])

    for key, val in [x for x in buildData['data'].iteritems()]:
        dataType = template.get(key, [mintdata.RDT_STRING])[0]
        if dataType == mintdata.RDT_INT:
            buildData['data'][key] = int(val)
        elif dataType == mintdata.RDT_BOOL:
            buildData['data'][key] = bool(int(val))

    timeSubmitted = time.time()
    timeStarted = None
    timeBuilt = None
    timeFinished = None
    print "submitting:", UUID
    mcpClient.submitJob(simplejson.dumps(buildData))

    status, statusMessage = None, None
    while status < jobstatus.BUILT:
        try:
            newStatus, newStatusMessage = mcpClient.jobStatus(UUID)
        except Exception, e:
            newStatus, newStatusMessage = jobstatus.ERROR, e
        if (newStatus != jobstatus.WAITING) and (status == jobstatus.WAITING):
            timeStarted = time.time()
        if (newStatus != status) or (statusMessage != newStatusMessage):
            status, statusMessage = newStatus, newStatusMessage
            print "%-79s\x0d" % ("%s: %s" % (jobstatus.statusName(status),
                                             statusMessage)),
            sys.stdout.flush()
        else:
            time.sleep(1)
    print ""

    if status == jobstatus.BUILT:
        timeBuilt = time.time()
        dataStr = outputQueue.read()
        data = simplejson.loads(dataStr)
        for url, type in data['urls']:
            os.system('wget %s -P %s' % (url, options.directory))
        mcpClient.stopJob(UUID)

    timeFinished = time.time()
    mcpClient.disconnect()
    outputQueue.disconnect()

    if not timeStarted:
        # waiting interval was so short it was not even measured.
        timeStarted = timeSubmitted

    print "Seconds waiting:", timeStarted - timeSubmitted
    if timeBuilt:
        print "Seconds building:", timeBuilt - timeStarted
        print "Seconds downloading:", timeFinished - timeBuilt
    print "Total:", timeFinished - timeStarted

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

_EXCLUDED_PATHS = (
    r"breakpad[\\\/].*",
    r"net/tools/spdyshark/[\\\/].*",
    r"skia[\\\/].*",
    r"v8[\\\/].*",
)

_TEXT_FILES = (
    r".*\.txt",
    r".*\.json",
)

_LICENSE_HEADER = (
     r".*? Copyright \(c\) 20[0-9\-]{2,7} The Chromium Authors\. All rights "
       r"reserved\." "\n"
     r".*? Use of this source code is governed by a BSD-style license that can "
       "be\n"
     r".*? found in the LICENSE file\."
       "\n"
)

def _CheckNoInterfacesInBase(input_api, output_api, source_file_filter):
  """Checks to make sure no files in libbase.a have |@interface|."""
  pattern = input_api.re.compile(r'@interface')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if (f.LocalPath().find('base/') != -1 and
        f.LocalPath().find('base/test/') == -1):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    return [ output_api.PresubmitError(
        'Objective-C interfaces or categories are forbidden in libbase. ' +
        'See http://groups.google.com/a/chromium.org/group/chromium-dev/' +
        'browse_thread/thread/efb28c10435987fd',
        files) ]
  return []

def _CheckSingletonInHeaders(input_api, output_api, source_file_filter):
  """Checks to make sure no header files have |Singleton<|."""
  pattern = input_api.re.compile(r'Singleton<')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if (f.LocalPath().endswith('.h') or f.LocalPath().endswith('.hxx') or
        f.LocalPath().endswith('.hpp') or f.LocalPath().endswith('.inl')):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    return [ output_api.PresubmitError(
        'Found Singleton<T> in the following header files.\n' +
        'Please move them to an appropriate source file so that the ' +
        'template gets instantiated in a single compilation unit.',
        files) ]
  return []

def _CheckConstNSObject(input_api, output_api, source_file_filter):
  """Checks to make sure no objective-c files have |const NSSomeClass*|."""
  pattern = input_api.re.compile(r'const\s+NS\w*\s*\*')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if f.LocalPath().endswith('.h') or f.LocalPath().endswith('.mm'):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    if input_api.is_committing:
      res_type = output_api.PresubmitPromptWarning
    else:
      res_type = output_api.PresubmitNotifyResult
    return [ res_type('|const NSClass*| is wrong, see ' +
                      'http://dev.chromium.org/developers/clang-mac',
                      files) ]
  return []

def _CommonChecks(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + _EXCLUDED_PATHS
  white_list = input_api.DEFAULT_WHITE_LIST + _TEXT_FILES
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  text_files = lambda x: input_api.FilterSourceFile(x, black_list=black_list,
                                                    white_list=white_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, source_file_filter=text_files))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckLicense(
      input_api, output_api, _LICENSE_HEADER, source_file_filter=sources))
  results.extend(_CheckConstNSObject(
      input_api, output_api, source_file_filter=sources))
  results.extend(_CheckSingletonInHeaders(
      input_api, output_api, source_file_filter=sources))
  results.extend(_CheckNoInterfacesInBase(
      input_api, output_api, source_file_filter=sources))
  return results

def CheckChangeOnUpload(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  if not input_api.json:
    results.append(output_api.PresubmitNotifyResult(
        'You don\'t have json nor simplejson installed.\n'
        '  This is a warning that you will need to upgrade your python '
        'installation.\n'
        '  This is no big deal but you\'ll eventually need to '
        'upgrade.\n'
        '  How? Easy! You can do it right now and shut me off! Just:\n'
        '    del depot_tools\\python.bat\n'
        '    gclient\n'
        '  Thanks for your patience.'))
  results.extend(_CommonChecks(input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  results.extend(input_api.canned_checks.CheckTreeIsOpen(
      input_api,
      output_api,
      json_url='http://chromium-status.appspot.com/current?format=json'))
  results.extend(input_api.canned_checks.CheckRietveldTryJobExecution(input_api,
      output_api, 'http://codereview.chromium.org', ('win', 'linux', 'mac'),
      'tryserver@chromium.org'))

  # These builders are just too slow.
  IGNORED_BUILDERS = [
    'Chromium XP',
    'Chromium Mac',
    'Chromium Arm (dbg)',
    'Chromium Linux',
    'Chromium Linux x64',
  ]
  results.extend(input_api.canned_checks.CheckBuildbotPendingBuilds(
      input_api,
      output_api,
      'http://build.chromium.org/p/chromium/json/builders?filter=1',
      6,
      IGNORED_BUILDERS))
  return results

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<commit_msg>Remove BUG= and TEST= checks from upload commit presubmit check.<commit_after># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

_EXCLUDED_PATHS = (
    r"^breakpad[\\\/].*",
    r"^net/tools/spdyshark/[\\\/].*",
    r"^skia[\\\/].*",
    r"^v8[\\\/].*",
    r".*MakeFile$",
)

_TEXT_FILES = (
    r".*\.txt",
    r".*\.json",
)

_LICENSE_HEADER = (
     r".*? Copyright \(c\) 20[0-9\-]{2,7} The Chromium Authors\. All rights "
       r"reserved\." "\n"
     r".*? Use of this source code is governed by a BSD-style license that can "
       "be\n"
     r".*? found in the LICENSE file\."
       "\n"
)

def _CheckNoInterfacesInBase(input_api, output_api, source_file_filter):
  """Checks to make sure no files in libbase.a have |@interface|."""
  pattern = input_api.re.compile(r'@interface')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if (f.LocalPath().find('base/') != -1 and
        f.LocalPath().find('base/test/') == -1):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    return [ output_api.PresubmitError(
        'Objective-C interfaces or categories are forbidden in libbase. ' +
        'See http://groups.google.com/a/chromium.org/group/chromium-dev/' +
        'browse_thread/thread/efb28c10435987fd',
        files) ]
  return []

def _CheckSingletonInHeaders(input_api, output_api, source_file_filter):
  """Checks to make sure no header files have |Singleton<|."""
  pattern = input_api.re.compile(r'Singleton<')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if (f.LocalPath().endswith('.h') or f.LocalPath().endswith('.hxx') or
        f.LocalPath().endswith('.hpp') or f.LocalPath().endswith('.inl')):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    return [ output_api.PresubmitError(
        'Found Singleton<T> in the following header files.\n' +
        'Please move them to an appropriate source file so that the ' +
        'template gets instantiated in a single compilation unit.',
        files) ]
  return []

def _CheckConstNSObject(input_api, output_api, source_file_filter):
  """Checks to make sure no objective-c files have |const NSSomeClass*|."""
  pattern = input_api.re.compile(r'const\s+NS\w*\s*\*')
  files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if f.LocalPath().endswith('.h') or f.LocalPath().endswith('.mm'):
      contents = input_api.ReadFile(f)
      if pattern.search(contents):
        files.append(f)

  if len(files):
    if input_api.is_committing:
      res_type = output_api.PresubmitPromptWarning
    else:
      res_type = output_api.PresubmitNotifyResult
    return [ res_type('|const NSClass*| is wrong, see ' +
                      'http://dev.chromium.org/developers/clang-mac',
                      files) ]
  return []

def _CommonChecks(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + _EXCLUDED_PATHS
  white_list = input_api.DEFAULT_WHITE_LIST + _TEXT_FILES
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  text_files = lambda x: input_api.FilterSourceFile(x, black_list=black_list,
                                                    white_list=white_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, source_file_filter=sources))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, source_file_filter=text_files))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckLicense(
      input_api, output_api, _LICENSE_HEADER, source_file_filter=sources))
  results.extend(_CheckConstNSObject(
      input_api, output_api, source_file_filter=sources))
  results.extend(_CheckSingletonInHeaders(
      input_api, output_api, source_file_filter=sources))
  results.extend(_CheckNoInterfacesInBase(
      input_api, output_api, source_file_filter=sources))
  return results

def CheckChangeOnUpload(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  if not input_api.json:
    results.append(output_api.PresubmitNotifyResult(
        'You don\'t have json nor simplejson installed.\n'
        '  This is a warning that you will need to upgrade your python '
        'installation.\n'
        '  This is no big deal but you\'ll eventually need to '
        'upgrade.\n'
        '  How? Easy! You can do it right now and shut me off! Just:\n'
        '    del depot_tools\\python.bat\n'
        '    gclient\n'
        '  Thanks for your patience.'))
  results.extend(_CommonChecks(input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  results.extend(input_api.canned_checks.CheckTreeIsOpen(
      input_api,
      output_api,
      json_url='http://chromium-status.appspot.com/current?format=json'))
  results.extend(input_api.canned_checks.CheckRietveldTryJobExecution(input_api,
      output_api, 'http://codereview.chromium.org', ('win', 'linux', 'mac'),
      'tryserver@chromium.org'))

  # These builders are just too slow.
  IGNORED_BUILDERS = [
    'Chromium XP',
    'Chromium Mac',
    'Chromium Arm (dbg)',
    'Chromium Linux',
    'Chromium Linux x64',
  ]
  results.extend(input_api.canned_checks.CheckBuildbotPendingBuilds(
      input_api,
      output_api,
      'http://build.chromium.org/p/chromium/json/builders?filter=1',
      6,
      IGNORED_BUILDERS))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  return results

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

class AboutSets(Koan):
    def test_sets_make_keep_lists_unique(self):
        highlanders = ['MacLeod', 'Ramirez', 'MacLeod', 'Matunas', 'MacLeod', 'Malcolm', 'MacLeod']

        there_can_only_be_only_one = set(highlanders)

        self.assertEqual(__, there_can_only_be_only_one)

    def test_empty_sets_have_different_syntax_to_populated_sets(self):
        self.assertEqual(__, {1, 2, 3})
        self.assertEqual(__, set())

    def test_dictionaries_and_sets_use_same_curly_braces(self):
        # Note: Literal sets using braces were introduced in python 3.
        #       They were also backported to python 2.7.

        self.assertEqual(__, {1, 2, 3}.__class__)
        self.assertEqual(__, {'one': 1, 'two': 2}.__class__)

        self.assertEqual(__, {}.__class__)

    def test_creating_sets_using_strings(self):
        self.assertEqual(__, {'12345'})
        self.assertEqual(__, set('12345'))

    def test_convert_the_set_into_a_list_to_sort_it(self):
        self.assertEqual(__, sorted(set('12345')))

    # ------------------------------------------------------------------

    def test_set_have_arithmetic_operators(self):
        scotsmen = {'MacLeod', 'Wallace', 'Willie'}
        warriors = {'MacLeod', 'Wallace', 'Leonidas'}

        self.assertEqual(__, scotsmen - warriors)
        self.assertEqual(__, scotsmen | warriors)
        self.assertEqual(__, scotsmen & warriors)
        self.assertEqual(__, scotsmen ^ warriors)

    # ------------------------------------------------------------------

    def test_we_can_query_set_membership(self):
        self.assertEqual(__, 127 in {127, 0, 0, 1} )
        self.assertEqual(__, 'cow' not in set('apocalypse now') )

    def test_we_can_compare_subsets(self):
        self.assertEqual(__, set('cake') <= set('cherry cake'))
        self.assertEqual(__, set('cake').issubset(set('cherry cake')) )

        self.assertEqual(__, set('cake') > set('pie'))
<commit_msg>[Python] Complete 'About Sets' koans<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

class AboutSets(Koan):
    def test_sets_make_keep_lists_unique(self):
        highlanders = ['MacLeod', 'Ramirez', 'MacLeod', 'Matunas', 'MacLeod', 'Malcolm', 'MacLeod']

        there_can_only_be_only_one = set(highlanders)

        self.assertEqual({'MacLeod', 'Ramirez', 'Matunas', 'Malcolm'}, there_can_only_be_only_one)

    def test_empty_sets_have_different_syntax_to_populated_sets(self):
        self.assertEqual(set([1,2,3]), {1, 2, 3})
        self.assertEqual(set([]), set())

    def test_dictionaries_and_sets_use_same_curly_braces(self):
        # Note: Literal sets using braces were introduced in python 3.
        #       They were also backported to python 2.7.

        self.assertEqual(set, {1, 2, 3}.__class__)
        self.assertEqual(dict, {'one': 1, 'two': 2}.__class__)

        self.assertEqual(dict, {}.__class__)

    def test_creating_sets_using_strings(self):
        self.assertEqual(set(['12345']), {'12345'})
        self.assertEqual({'1', '2', '3', '4', '5'}, set('12345'))

    def test_convert_the_set_into_a_list_to_sort_it(self):
        self.assertEqual(['1', '2', '3', '4', '5'], sorted(set('12345')))

    # ------------------------------------------------------------------

    def test_set_have_arithmetic_operators(self):
        scotsmen = {'MacLeod', 'Wallace', 'Willie'}
        warriors = {'MacLeod', 'Wallace', 'Leonidas'}

        self.assertEqual({'Willie'}, scotsmen - warriors)
        self.assertEqual({'MacLeod', 'Wallace', 'Willie', 'Leonidas'}, scotsmen | warriors)
        self.assertEqual({'MacLeod', 'Wallace'}, scotsmen & warriors)
        self.assertEqual({'Willie', 'Leonidas'}, scotsmen ^ warriors)

    # ------------------------------------------------------------------

    def test_we_can_query_set_membership(self):
        self.assertEqual(True, 127 in {127, 0, 0, 1} )
        self.assertEqual(True, 'cow' not in set('apocalypse now') )

    def test_we_can_compare_subsets(self):
        self.assertEqual(True, set('cake') <= set('cherry cake'))
        self.assertEqual(True, set('cake').issubset(set('cherry cake')) )

        self.assertEqual(False, set('cake') > set('pie'))
<|endoftext|>


<commit_before>from collections import deque
import glob
import hashlib
import os.path
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from libearth.compat import binary, binary_type
from libearth.feed import Feed
from libearth.feedlist import (Feed as FeedOutline,
                               FeedCategory as CategoryOutline, FeedList)
from libearth.parser.autodiscovery import autodiscovery, FeedUrlNotFoundError
from libearth.parser.heuristic import get_format
from libearth.schema import read, write

from flask import Flask, jsonify, render_template, request, url_for

app = Flask(__name__)

app.config.update(dict(
    REPOSITORY='repo/',
    OPML='earthreader.opml'
))

@app.route('/', methods=['GET'])
def index():
    return render_template('index.html')

def get_all_feeds(category, parent_categories=[]):
    result = []
    categories = []
    if parent_categories:
        feed_path = '/'.join(parent_categories)
    else:
        feed_path = '/'
    for child in category:
        if isinstance(child, FeedOutline):
            feed_id = hashlib.sha1(child.xml_url).hexdigest()
            result.append({
                'title': child.title,
                'feed_url': url_for(
                    'category_feed_entries',
                    category_id=feed_path,
                    feed_id=feed_id
                )
            })
        elif isinstance(child, CategoryOutline):
            categories.append(child)
    for category in categories:
        result.append({
            'title': category.title,
            'feed_url': feed_path + category.title + '/feeds/',
            'feeds': get_all_feeds(
                category,
                parent_categories.append(category.title)
                if parent_categories else [category.title]
            )
        })
    return result

def check_path_valid(category_id):
    REPOSITORY = app.config['REPOSITORY']
    OPML = app.config['OPML']
    categories = deque(category_id.split('/'))
    cursor = FeedList(REPOSITORY + OPML)
    while categories:
        is_searched = False
        looking_for = categories.popleft()
        for category in cursor:
            if category.text == looking_for:
                is_searched = True
                cursor = category
                break
        if not is_searched:
            return False
    return True

@app.route('/feeds/', methods=['GET'])
def feeds():
    REPOSITORY = app.config['REPOSITORY']
    OPML = app.config['OPML']
    if not os.path.isfile(REPOSITORY + OPML):
        r = jsonify(
            error='opml-not-found',
            message='Cannot open OPML'
        )
        r.status_code = 400
        return r
    feed_list = FeedList(REPOSITORY + OPML)
    feeds = get_all_feeds(feed_list, None)
    return jsonify(feeds=feeds)

@app.route('/feeds/<feed_id>/entries/')
def feed_entries(feed_id):
    REPOSITORY = app.config['REPOSITORY']
    try:
        with open(os.path.join(REPOSITORY, feed_id + '.xml')) as f:
            feed = read(Feed, f)
            entries = []
            for entry in feed.entries:
                entries.append({
                    'title': entry.title,
                    'entry_url': url_for(
                        'feed_entry',
                        feed_id=feed_id,
                        entry_id=hashlib.sha1(binary(entry.id)).hexdigest(),
                        _external=True
                    ),
                    'updated': entry.published_at
                })
        return jsonify(
            title=feed.title,
            entries=entries
        )
    except IOError:
        r = jsonify(
            error='feed-not-found',
            message='Given feed does not exist'
        )
        r.status_code = 404
        return r

@app.route('/<path:category_id>/feeds/<feed_id>/entries/')
def category_feed_entries(category_id, feed_id):
    if check_path_valid(category_id):
        return feed_entries(feed_id)
    else:
        r = jsonify(
            error='category-path-invalid',
            message='Given category path is not valid'
        )
        r.status_code = 404
        return r

@app.route('/feeds/<feed_id>/entries/<entry_id>/')
def feed_entry(feed_id, entry_id):
    REPOSITORY = app.config['REPOSITORY']
    try:
        with open(os.path.join(REPOSITORY, feed_id + '.xml')) as f:
            feed = read(Feed, f)
            for entry in feed.entries:
                if entry_id == hashlib.sha1(binary(entry.id)).hexdigest():
                    return jsonify(
                        content=entry.content,
                        updated=binary_type(entry.updated_at)
                    )
            r = jsonify(
                error='entry-not-found',
                message='Given entry does not exist'
            )
            r.status_code = 404
            return r

    except IOError:
        r = jsonify(
            error='feed-not-found',
            message='Given feed does not exist'
        )
        r.status_code = 404
        return r

@app.route('/<path:category_id>/feeds/<feed_id>/entries/<entry_id>/')
def category_feed_entry(category_id, feed_id, entry_id):
    if check_path_valid(category_id):
        return feed_entry(feed_id, entry_id)
    else:
        r = jsonify(
            error='category-path-invalid',
            message='Given category path is not valid'
        )
        r.status_code = 404
        return r
<commit_msg>Check_path_valid returns a cursor to use it when add a feed in category<commit_after>from collections import deque
import glob
import hashlib
import os.path
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from libearth.compat import binary, binary_type
from libearth.feed import Feed
from libearth.feedlist import (Feed as FeedOutline,
                               FeedCategory as CategoryOutline, FeedList)
from libearth.parser.autodiscovery import autodiscovery, FeedUrlNotFoundError
from libearth.parser.heuristic import get_format
from libearth.schema import read, write

from flask import Flask, jsonify, render_template, request, url_for

app = Flask(__name__)

app.config.update(dict(
    REPOSITORY='repo/',
    OPML='earthreader.opml'
))

@app.route('/', methods=['GET'])
def index():
    return render_template('index.html')

def get_all_feeds(category, parent_categories=[]):
    result = []
    categories = []
    if parent_categories:
        feed_path = '/'.join(parent_categories)
    else:
        feed_path = '/'
    for child in category:
        if isinstance(child, FeedOutline):
            feed_id = hashlib.sha1(child.xml_url).hexdigest()
            result.append({
                'title': child.title,
                'feed_url': url_for(
                    'category_feed_entries',
                    category_id=feed_path,
                    feed_id=feed_id
                )
            })
        elif isinstance(child, CategoryOutline):
            categories.append(child)
    for category in categories:
        result.append({
            'title': category.title,
            'feed_url': feed_path + category.title + '/feeds/',
            'feeds': get_all_feeds(
                category,
                parent_categories.append(category.title)
                if parent_categories else [category.title]
            )
        })
    return result

def check_path_valid(category_id):
    REPOSITORY = app.config['REPOSITORY']
    OPML = app.config['OPML']
    categories = deque(category_id.split('/'))
    cursor = FeedList(REPOSITORY + OPML)
    while categories:
        is_searched = False
        looking_for = categories.popleft()
        for category in cursor:
            if category.text == looking_for:
                is_searched = True
                cursor = category
                break
        if not is_searched:
            return None
    return cursor

@app.route('/feeds/', methods=['GET'])
def feeds():
    REPOSITORY = app.config['REPOSITORY']
    OPML = app.config['OPML']
    if not os.path.isfile(REPOSITORY + OPML):
        r = jsonify(
            error='opml-not-found',
            message='Cannot open OPML'
        )
        r.status_code = 400
        return r
    feed_list = FeedList(REPOSITORY + OPML)
    feeds = get_all_feeds(feed_list, None)
    return jsonify(feeds=feeds)

@app.route('/feeds/<feed_id>/entries/')
def feed_entries(feed_id):
    REPOSITORY = app.config['REPOSITORY']
    try:
        with open(os.path.join(REPOSITORY, feed_id + '.xml')) as f:
            feed = read(Feed, f)
            entries = []
            for entry in feed.entries:
                entries.append({
                    'title': entry.title,
                    'entry_url': url_for(
                        'feed_entry',
                        feed_id=feed_id,
                        entry_id=hashlib.sha1(binary(entry.id)).hexdigest(),
                        _external=True
                    ),
                    'updated': entry.published_at
                })
        return jsonify(
            title=feed.title,
            entries=entries
        )
    except IOError:
        r = jsonify(
            error='feed-not-found',
            message='Given feed does not exist'
        )
        r.status_code = 404
        return r

@app.route('/<path:category_id>/feeds/<feed_id>/entries/')
def category_feed_entries(category_id, feed_id):
    if check_path_valid(category_id):
        return feed_entries(feed_id)
    else:
        r = jsonify(
            error='category-path-invalid',
            message='Given category path is not valid'
        )
        r.status_code = 404
        return r

@app.route('/feeds/<feed_id>/entries/<entry_id>/')
def feed_entry(feed_id, entry_id):
    REPOSITORY = app.config['REPOSITORY']
    try:
        with open(os.path.join(REPOSITORY, feed_id + '.xml')) as f:
            feed = read(Feed, f)
            for entry in feed.entries:
                if entry_id == hashlib.sha1(binary(entry.id)).hexdigest():
                    return jsonify(
                        content=entry.content,
                        updated=binary_type(entry.updated_at)
                    )
            r = jsonify(
                error='entry-not-found',
                message='Given entry does not exist'
            )
            r.status_code = 404
            return r

    except IOError:
        r = jsonify(
            error='feed-not-found',
            message='Given feed does not exist'
        )
        r.status_code = 404
        return r

@app.route('/<path:category_id>/feeds/<feed_id>/entries/<entry_id>/')
def category_feed_entry(category_id, feed_id, entry_id):
    if check_path_valid(category_id):
        return feed_entry(feed_id, entry_id)
    else:
        r = jsonify(
            error='category-path-invalid',
            message='Given category path is not valid'
        )
        r.status_code = 404
        return r
<|endoftext|>


<commit_before>"""celery.worker"""
from carrot.connection import DjangoAMQPConnection
from celery.messaging import TaskConsumer
from celery.conf import DAEMON_CONCURRENCY, DAEMON_LOG_FILE
from celery.conf import QUEUE_WAKEUP_AFTER, EMPTY_MSG_EMIT_EVERY
from celery.conf import SEND_CELERY_TASK_ERROR_EMAILS
from celery.log import setup_logger
from celery.registry import tasks
from celery.pool import TaskPool
from celery.datastructures import ExceptionInfo
from celery.backends import default_backend, default_periodic_status_backend
from celery.timer import EventTimer
from django.core.mail import mail_admins
import multiprocessing
import traceback
import threading
import logging
import socket
import time
import sys

# pep8.py borks on a inline signature separator and
# says "trailing whitespace" ;)
EMAIL_SIGNATURE_SEP = "-- "
TASK_FAIL_EMAIL_BODY = """
Task %%(name)s with id %%(id)s raised exception: %%(exc)s

The contents of the full traceback was:

%%(traceback)s

%%(EMAIL_SIGNATURE_SEP)s
Just thought I'd let you know!
celeryd at %%(hostname)s.
""" % {"EMAIL_SIGNATURE_SEP": EMAIL_SIGNATURE_SEP}

class EmptyQueue(Exception):
    """The message queue is currently empty."""

class UnknownTask(Exception):
    """Got an unknown task in the queue. The message is requeued and
    ignored."""

def jail(task_id, func, args, kwargs):
    """Wraps the task in a jail, which catches all exceptions, and
    saves the status and result of the task execution to the task
    meta backend.

    If the call was successful, it saves the result to the task result
    backend, and sets the task status to ``"DONE"``.

    If the call results in an exception, it saves the exception as the task
    result, and sets the task status to ``"FAILURE"``.

    :param task_id: The id of the task.
    :param func: Callable object to execute.
    :param args: List of positional args to pass on to the function.
    :param kwargs: Keyword arguments mapping to pass on to the function.

    :returns: the function return value on success, or
        the exception instance on failure.

    """

    # See: http://groups.google.com/group/django-users/browse_thread/
    #       thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst&
    #       q=multiprocessing#38402e76cf3233e8
    from django.db import connection
    connection.close()

    # Reset cache connection only if using memcached
    from django.core import cache
    if cache.parse_backend_uri(cache.settings.CACHE_BACKEND)[0] == 'memcached':
        cache.cache.close()

    # Backend process cleanup
    default_backend.process_cleanup()

    # Convert any unicode keys in the keyword arguments to ascii.
    kwargs = dict([(k.encode("utf-8"), v)
                        for k, v in kwargs.items()])
    try:
        result = func(*args, **kwargs)
    except Exception, exc:
        default_backend.mark_as_failure(task_id, exc)
        return ExceptionInfo(sys.exc_info())
    else:
        default_backend.mark_as_done(task_id, result)
        return result

class TaskWrapper(object):
    """Class wrapping a task to be run.

    :param task_name: see :attr:`task_name`.

    :param task_id: see :attr:`task_id`.

    :param task_func: see :attr:`task_func`

    :param args: see :attr:`args`

    :param kwargs: see :attr:`kwargs`.

    .. attribute:: task_name

        Kind of task. Must be a name registered in the task registry.

    .. attribute:: task_id

        UUID of the task.

    .. attribute:: task_func

        The tasks callable object.

    .. attribute:: args

        List of positional arguments to apply to the task.

    .. attribute:: kwargs

        Mapping of keyword arguments to apply to the task.

    """
    success_msg = "Task %(name)s[%(id)s] processed: %(return_value)s"
    fail_msg = """
        Task %(name)s[%(id)s] raised exception: %(exc)s\n%(traceback)s
    """
    fail_email_subject = """
        [celery@%(hostname)s] Error: Task %(name)s (%(id)s): %(exc)s
    """
    fail_email_body = TASK_FAIL_EMAIL_BODY

    def __init__(self, task_name, task_id, task_func, args, kwargs, **opts):
        self.task_name = task_name
        self.task_id = task_id
        self.task_func = task_func
        self.args = args
        self.kwargs = kwargs
        self.logger = kwargs.get("logger")
        for opt in ("success_msg", "fail_msg", "fail_email_subject",
                "fail_email_body"):
            setattr(self, opt, opts.get(opt, getattr(self, opt, None)))
        if not self.logger:
            self.logger = multiprocessing.get_logger()

    def __repr__(self):
        return '<%s: {name:"%s", id:"%s", args:"%s", kwargs:"%s"}>' % (
                self.__class__.__name__,
                self.task_name, self.task_id,
                self.args, self.kwargs)

    @classmethod
    def from_message(cls, message, logger):
        """Create a :class:`TaskWrapper` from a task message sent by
        :class:`celery.messaging.TaskPublisher`.

        :raises UnknownTask: if the message does not describe a task,
            the message is also rejected.

        :returns: :class:`TaskWrapper` instance.

        """
        message_data = message.decode()
        task_name = message_data["task"]
        task_id = message_data["id"]
        args = message_data["args"]
        kwargs = message_data["kwargs"]
        if task_name not in tasks:
            raise UnknownTask(task_name)
        task_func = tasks[task_name]
        return cls(task_name, task_id, task_func, args, kwargs, logger=logger)

    def extend_with_default_kwargs(self, loglevel, logfile):
        """Extend the tasks keyword arguments with standard task arguments.

        These are ``logfile``, ``loglevel``, ``task_id`` and ``task_name``.

        """
        task_func_kwargs = {"logfile": logfile,
                            "loglevel": loglevel,
                            "task_id": self.task_id,
                            "task_name": self.task_name}
        task_func_kwargs.update(self.kwargs)
        return task_func_kwargs

    def execute(self, loglevel=None, logfile=None):
        """Execute the task in a :func:`jail` and store return value
        and status in the task meta backend.

        :keyword loglevel: The loglevel used by the task.

        :keyword logfile: The logfile used by the task.

        """
        task_func_kwargs = self.extend_with_default_kwargs(loglevel, logfile)
        return jail(self.task_id, [
                        self.task_func, self.args, task_func_kwargs])

    def on_success(self, ret_value, meta):
        """The handler used if the task was successfully processed (
        without raising an exception)."""
        task_id = meta.get("task_id")
        task_name = meta.get("task_name")
        msg = self.success_msg.strip() % {
                "id": task_id,
                "name": task_name,
                "return_value": ret_value}
        self.logger.info(msg)

    def on_failure(self, exc_info, meta):
        """The handler used if the task raised an exception."""
        task_id = meta.get("task_id")
        task_name = meta.get("task_name")
        context = {
            "hostname": socket.gethostname(),
            "id": task_id,
            "name": task_name,
            "exc": exc_info.exception,
            "traceback": exc_info.traceback,
        }
        self.logger.error(self.fail_msg.strip() % context)

        if SEND_CELERY_TASK_ERROR_EMAILS:
            subject = self.fail_email_subject.strip() % context
            body = self.fail_email_body.strip() % context
            mail_admins(subject, body, fail_silently=True)

    def execute_using_pool(self, pool, loglevel=None, logfile=None):
        """Like :meth:`execute`, but using the :mod:`multiprocessing` pool.

        :param pool: A :class:`multiprocessing.Pool` instance.

        :keyword loglevel: The loglevel used by the task.

        :keyword logfile: The logfile used by the task.

        :returns :class:`multiprocessing.AsyncResult` instance.

        """
        task_func_kwargs = self.extend_with_default_kwargs(loglevel, logfile)
        jail_args = [self.task_id, self.task_func,
                     self.args, task_func_kwargs]
        return pool.apply_async(jail, args=jail_args,
                callbacks=[self.on_success], errbacks=[self.on_failure],
                meta={"task_id": self.task_id, "task_name": self.task_name})

class PeriodicWorkController(threading.Thread):
    """A thread that continuously checks if there are
    :class:`celery.task.PeriodicTask`s waiting for execution, and executes
    them.

    Example

        >>> PeriodicWorkController().start()

    """

    def run(self):
        """Don't use :meth:`run`. use :meth:`start`."""
        while True:
            default_periodic_status_backend.run_periodic_tasks()
            time.sleep(1)

class WorkController(object):
    """Executes tasks waiting in the task queue.

    :param concurrency: see :attr:`concurrency`.

    :param logfile: see :attr:`logfile`.

    :param loglevel: see :attr:`loglevel`.

    :param queue_wakeup_after: see :attr:`queue_wakeup_after`.

    .. attribute:: concurrency

        The number of simultaneous processes doing work (default:
        :const:`celery.conf.DAEMON_CONCURRENCY`)

    .. attribute:: loglevel

        The loglevel used (default: :const:`logging.INFO`)

    .. attribute:: logfile

        The logfile used, if no logfile is specified it uses ``stderr``
        (default: :const:`celery.conf.DAEMON_LOG_FILE`).

    .. attribute:: queue_wakeup_after

        The time it takes for the daemon to wake up after the queue is empty,
        so it can check for more work
        (default: :const:`celery.conf.QUEUE_WAKEUP_AFTER`).

    .. attribute:: empty_msg_emit_every

        How often the daemon emits the ``"Waiting for queue..."`` message.
        If this is ``None``, the message will never be logged.
        (default: :const:`celery.conf.EMPTY_MSG_EMIT_EVERY`)

    .. attribute:: logger

        The :class:`logging.Logger` instance used for logging.

    .. attribute:: pool

        The :class:`multiprocessing.Pool` instance used.

    .. attribute:: task_consumer

        The :class:`celery.messaging.TaskConsumer` instance used.

    """
    loglevel = logging.ERROR
    concurrency = DAEMON_CONCURRENCY
    logfile = DAEMON_LOG_FILE
    queue_wakeup_after = QUEUE_WAKEUP_AFTER
    empty_msg_emit_every = EMPTY_MSG_EMIT_EVERY

    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            queue_wakeup_after=None, is_detached=False):
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.queue_wakeup_after = queue_wakeup_after or \
                                    self.queue_wakeup_after
        self.logger = setup_logger(loglevel, logfile)
        self.pool = TaskPool(self.concurrency, logger=self.logger)
        self.task_consumer = None
        self.task_consumer_it = None
        self.is_detached = is_detached
        self.reset_connection()

    def reset_connection(self):
        """Reset the AMQP connection, and reinitialize the
        :class:`celery.messaging.TaskConsumer` instance.

        Resets the task consumer in :attr:`task_consumer`.

        """
        if self.task_consumer:
            self.task_consumer.connection.close()
        amqp_connection = DjangoAMQPConnection()
        self.task_consumer = TaskConsumer(connection=amqp_connection)
        self.task_consumer_it = self.task_consumer.iterqueue(infinite=True)

    def connection_diagnostics(self):
        """Diagnose the AMQP connection, and reset connection if
        necessary."""
        connection = self.task_consumer.backend.channel.connection

        if not connection:
            self.logger.info(
                    "AMQP Connection has died, restoring connection.")
            self.reset_connection()

    def receive_message(self):
        """Receive the next message from the message broker.

        Tries to reset the AMQP connection if not available.
        Returns ``None`` if no message is waiting on the queue.

        :rtype: :class:`carrot.messaging.Message` instance.

        """
        message = self.task_consumer_it.next()
        if not message:
            raise EmptyQueue()
        return message

    def process_task(self, message):
        """Process task message by passing it to the pool of workers."""
        task = TaskWrapper.from_message(message, logger=self.logger)
        self.logger.info("Got task from broker: %s[%s]" % (
            task.task_name, task.task_id))
        self.logger.debug("Got a task: %s. Trying to execute it..." % task)

        result = task.execute_using_pool(self.pool, self.loglevel,
                                         self.logfile)

        self.logger.debug("Task %s has been executed asynchronously." % task)

        return result

    def execute_next_task(self):
        """Execute the next task on the queue using the multiprocessing pool.

        Catches all exceptions and logs them with level
        :const:`logging.CRITICAL`.

        Raises :exc:`EmptyQueue` exception if there is no message
        waiting on the queue.

        """
        self.process_task(self.receive_message())

    def schedule_retry_tasks(self):
        """Reschedule all requeued tasks waiting for retry."""
        pass

    def run(self):
        """Starts the workers main loop."""
        log_wait = lambda: self.logger.info("Waiting for queue...")
        ev_msg_waiting = EventTimer(log_wait, self.empty_msg_emit_every)

        self.pool.run()
        PeriodicWorkController().start()

        # If not running as daemon, and DEBUG logging level is enabled,
        # print pool PIDs and sleep for a second before we start.
        if self.logger.isEnabledFor(logging.DEBUG):
            self.logger.debug("Pool child processes: [%s]" % (
                "|".join(map(str, self.pool.get_worker_pids()))))
            if not self.is_detached:
                time.sleep(1)

        while True:
            try:
                self.execute_next_task()
            except ValueError:
                # execute_next_task didn't return a r/name/id tuple,
                # probably because it got an exception.
                continue
            except EmptyQueue:
                ev_msg_waiting.tick()
                time.sleep(self.queue_wakeup_after)
                continue
            except UnknownTask, exc:
                self.logger.info("Unknown task ignored: %s" % (exc))
                continue
            except Exception, exc:
                self.logger.critical("Message queue raised %s: %s\n%s" % (
                             exc.__class__, exc, traceback.format_exc()))
                continue
<commit_msg>Can't exact match for "memcached". Search for it as a substring instead. At Opera we use a custom cache backend that uses libmemcached instead of libmemcache (cmemcache). Should find a better solution for this, but for now "memcached" should probably be unique enough of a string to not make problems.<commit_after>"""celery.worker"""
from carrot.connection import DjangoAMQPConnection
from celery.messaging import TaskConsumer
from celery.conf import DAEMON_CONCURRENCY, DAEMON_LOG_FILE
from celery.conf import QUEUE_WAKEUP_AFTER, EMPTY_MSG_EMIT_EVERY
from celery.conf import SEND_CELERY_TASK_ERROR_EMAILS
from celery.log import setup_logger
from celery.registry import tasks
from celery.pool import TaskPool
from celery.datastructures import ExceptionInfo
from celery.backends import default_backend, default_periodic_status_backend
from celery.timer import EventTimer
from django.core.mail import mail_admins
import multiprocessing
import traceback
import threading
import logging
import socket
import time
import sys

# pep8.py borks on a inline signature separator and
# says "trailing whitespace" ;)
EMAIL_SIGNATURE_SEP = "-- "
TASK_FAIL_EMAIL_BODY = """
Task %%(name)s with id %%(id)s raised exception: %%(exc)s

The contents of the full traceback was:

%%(traceback)s

%%(EMAIL_SIGNATURE_SEP)s
Just thought I'd let you know!
celeryd at %%(hostname)s.
""" % {"EMAIL_SIGNATURE_SEP": EMAIL_SIGNATURE_SEP}

class EmptyQueue(Exception):
    """The message queue is currently empty."""

class UnknownTask(Exception):
    """Got an unknown task in the queue. The message is requeued and
    ignored."""

def jail(task_id, func, args, kwargs):
    """Wraps the task in a jail, which catches all exceptions, and
    saves the status and result of the task execution to the task
    meta backend.

    If the call was successful, it saves the result to the task result
    backend, and sets the task status to ``"DONE"``.

    If the call results in an exception, it saves the exception as the task
    result, and sets the task status to ``"FAILURE"``.

    :param task_id: The id of the task.
    :param func: Callable object to execute.
    :param args: List of positional args to pass on to the function.
    :param kwargs: Keyword arguments mapping to pass on to the function.

    :returns: the function return value on success, or
        the exception instance on failure.

    """

    # See: http://groups.google.com/group/django-users/browse_thread/
    #       thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst&
    #       q=multiprocessing#38402e76cf3233e8
    from django.db import connection
    connection.close()

    # Reset cache connection only if using memcached/libmemcached
    from django.core import cache
    # XXX At Opera we use a custom memcached backend that uses libmemcached
    # instead of libmemcache (cmemcache). Should find a better solution for
    # this, but for now "memcached" should probably be unique enough of a
    # string to not make problems.
    if "memcached" in cache.parse_backend_uri(settings.CACHE_BACKEND)[0]:
        cache.cache.close()

    # Backend process cleanup
    default_backend.process_cleanup()

    # Convert any unicode keys in the keyword arguments to ascii.
    kwargs = dict([(k.encode("utf-8"), v)
                        for k, v in kwargs.items()])
    try:
        result = func(*args, **kwargs)
    except Exception, exc:
        default_backend.mark_as_failure(task_id, exc)
        return ExceptionInfo(sys.exc_info())
    else:
        default_backend.mark_as_done(task_id, result)
        return result

class TaskWrapper(object):
    """Class wrapping a task to be run.

    :param task_name: see :attr:`task_name`.

    :param task_id: see :attr:`task_id`.

    :param task_func: see :attr:`task_func`

    :param args: see :attr:`args`

    :param kwargs: see :attr:`kwargs`.

    .. attribute:: task_name

        Kind of task. Must be a name registered in the task registry.

    .. attribute:: task_id

        UUID of the task.

    .. attribute:: task_func

        The tasks callable object.

    .. attribute:: args

        List of positional arguments to apply to the task.

    .. attribute:: kwargs

        Mapping of keyword arguments to apply to the task.

    """
    success_msg = "Task %(name)s[%(id)s] processed: %(return_value)s"
    fail_msg = """
        Task %(name)s[%(id)s] raised exception: %(exc)s\n%(traceback)s
    """
    fail_email_subject = """
        [celery@%(hostname)s] Error: Task %(name)s (%(id)s): %(exc)s
    """
    fail_email_body = TASK_FAIL_EMAIL_BODY

    def __init__(self, task_name, task_id, task_func, args, kwargs, **opts):
        self.task_name = task_name
        self.task_id = task_id
        self.task_func = task_func
        self.args = args
        self.kwargs = kwargs
        self.logger = kwargs.get("logger")
        for opt in ("success_msg", "fail_msg", "fail_email_subject",
                "fail_email_body"):
            setattr(self, opt, opts.get(opt, getattr(self, opt, None)))
        if not self.logger:
            self.logger = multiprocessing.get_logger()

    def __repr__(self):
        return '<%s: {name:"%s", id:"%s", args:"%s", kwargs:"%s"}>' % (
                self.__class__.__name__,
                self.task_name, self.task_id,
                self.args, self.kwargs)

    @classmethod
    def from_message(cls, message, logger):
        """Create a :class:`TaskWrapper` from a task message sent by
        :class:`celery.messaging.TaskPublisher`.

        :raises UnknownTask: if the message does not describe a task,
            the message is also rejected.

        :returns: :class:`TaskWrapper` instance.

        """
        message_data = message.decode()
        task_name = message_data["task"]
        task_id = message_data["id"]
        args = message_data["args"]
        kwargs = message_data["kwargs"]
        if task_name not in tasks:
            raise UnknownTask(task_name)
        task_func = tasks[task_name]
        return cls(task_name, task_id, task_func, args, kwargs, logger=logger)

    def extend_with_default_kwargs(self, loglevel, logfile):
        """Extend the tasks keyword arguments with standard task arguments.

        These are ``logfile``, ``loglevel``, ``task_id`` and ``task_name``.

        """
        task_func_kwargs = {"logfile": logfile,
                            "loglevel": loglevel,
                            "task_id": self.task_id,
                            "task_name": self.task_name}
        task_func_kwargs.update(self.kwargs)
        return task_func_kwargs

    def execute(self, loglevel=None, logfile=None):
        """Execute the task in a :func:`jail` and store return value
        and status in the task meta backend.

        :keyword loglevel: The loglevel used by the task.

        :keyword logfile: The logfile used by the task.

        """
        task_func_kwargs = self.extend_with_default_kwargs(loglevel, logfile)
        return jail(self.task_id, [
                        self.task_func, self.args, task_func_kwargs])

    def on_success(self, ret_value, meta):
        """The handler used if the task was successfully processed (
        without raising an exception)."""
        task_id = meta.get("task_id")
        task_name = meta.get("task_name")
        msg = self.success_msg.strip() % {
                "id": task_id,
                "name": task_name,
                "return_value": ret_value}
        self.logger.info(msg)

    def on_failure(self, exc_info, meta):
        """The handler used if the task raised an exception."""
        task_id = meta.get("task_id")
        task_name = meta.get("task_name")
        context = {
            "hostname": socket.gethostname(),
            "id": task_id,
            "name": task_name,
            "exc": exc_info.exception,
            "traceback": exc_info.traceback,
        }
        self.logger.error(self.fail_msg.strip() % context)

        if SEND_CELERY_TASK_ERROR_EMAILS:
            subject = self.fail_email_subject.strip() % context
            body = self.fail_email_body.strip() % context
            mail_admins(subject, body, fail_silently=True)

    def execute_using_pool(self, pool, loglevel=None, logfile=None):
        """Like :meth:`execute`, but using the :mod:`multiprocessing` pool.

        :param pool: A :class:`multiprocessing.Pool` instance.

        :keyword loglevel: The loglevel used by the task.

        :keyword logfile: The logfile used by the task.

        :returns :class:`multiprocessing.AsyncResult` instance.

        """
        task_func_kwargs = self.extend_with_default_kwargs(loglevel, logfile)
        jail_args = [self.task_id, self.task_func,
                     self.args, task_func_kwargs]
        return pool.apply_async(jail, args=jail_args,
                callbacks=[self.on_success], errbacks=[self.on_failure],
                meta={"task_id": self.task_id, "task_name": self.task_name})

class PeriodicWorkController(threading.Thread):
    """A thread that continuously checks if there are
    :class:`celery.task.PeriodicTask`s waiting for execution, and executes
    them.

    Example

        >>> PeriodicWorkController().start()

    """

    def run(self):
        """Don't use :meth:`run`. use :meth:`start`."""
        while True:
            default_periodic_status_backend.run_periodic_tasks()
            time.sleep(1)

class WorkController(object):
    """Executes tasks waiting in the task queue.

    :param concurrency: see :attr:`concurrency`.

    :param logfile: see :attr:`logfile`.

    :param loglevel: see :attr:`loglevel`.

    :param queue_wakeup_after: see :attr:`queue_wakeup_after`.

    .. attribute:: concurrency

        The number of simultaneous processes doing work (default:
        :const:`celery.conf.DAEMON_CONCURRENCY`)

    .. attribute:: loglevel

        The loglevel used (default: :const:`logging.INFO`)

    .. attribute:: logfile

        The logfile used, if no logfile is specified it uses ``stderr``
        (default: :const:`celery.conf.DAEMON_LOG_FILE`).

    .. attribute:: queue_wakeup_after

        The time it takes for the daemon to wake up after the queue is empty,
        so it can check for more work
        (default: :const:`celery.conf.QUEUE_WAKEUP_AFTER`).

    .. attribute:: empty_msg_emit_every

        How often the daemon emits the ``"Waiting for queue..."`` message.
        If this is ``None``, the message will never be logged.
        (default: :const:`celery.conf.EMPTY_MSG_EMIT_EVERY`)

    .. attribute:: logger

        The :class:`logging.Logger` instance used for logging.

    .. attribute:: pool

        The :class:`multiprocessing.Pool` instance used.

    .. attribute:: task_consumer

        The :class:`celery.messaging.TaskConsumer` instance used.

    """
    loglevel = logging.ERROR
    concurrency = DAEMON_CONCURRENCY
    logfile = DAEMON_LOG_FILE
    queue_wakeup_after = QUEUE_WAKEUP_AFTER
    empty_msg_emit_every = EMPTY_MSG_EMIT_EVERY

    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            queue_wakeup_after=None, is_detached=False):
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.queue_wakeup_after = queue_wakeup_after or \
                                    self.queue_wakeup_after
        self.logger = setup_logger(loglevel, logfile)
        self.pool = TaskPool(self.concurrency, logger=self.logger)
        self.task_consumer = None
        self.task_consumer_it = None
        self.is_detached = is_detached
        self.reset_connection()

    def reset_connection(self):
        """Reset the AMQP connection, and reinitialize the
        :class:`celery.messaging.TaskConsumer` instance.

        Resets the task consumer in :attr:`task_consumer`.

        """
        if self.task_consumer:
            self.task_consumer.connection.close()
        amqp_connection = DjangoAMQPConnection()
        self.task_consumer = TaskConsumer(connection=amqp_connection)
        self.task_consumer_it = self.task_consumer.iterqueue(infinite=True)

    def connection_diagnostics(self):
        """Diagnose the AMQP connection, and reset connection if
        necessary."""
        connection = self.task_consumer.backend.channel.connection

        if not connection:
            self.logger.info(
                    "AMQP Connection has died, restoring connection.")
            self.reset_connection()

    def receive_message(self):
        """Receive the next message from the message broker.

        Tries to reset the AMQP connection if not available.
        Returns ``None`` if no message is waiting on the queue.

        :rtype: :class:`carrot.messaging.Message` instance.

        """
        message = self.task_consumer_it.next()
        if not message:
            raise EmptyQueue()
        return message

    def process_task(self, message):
        """Process task message by passing it to the pool of workers."""
        task = TaskWrapper.from_message(message, logger=self.logger)
        self.logger.info("Got task from broker: %s[%s]" % (
            task.task_name, task.task_id))
        self.logger.debug("Got a task: %s. Trying to execute it..." % task)

        result = task.execute_using_pool(self.pool, self.loglevel,
                                         self.logfile)

        self.logger.debug("Task %s has been executed asynchronously." % task)

        return result

    def execute_next_task(self):
        """Execute the next task on the queue using the multiprocessing pool.

        Catches all exceptions and logs them with level
        :const:`logging.CRITICAL`.

        Raises :exc:`EmptyQueue` exception if there is no message
        waiting on the queue.

        """
        self.process_task(self.receive_message())

    def schedule_retry_tasks(self):
        """Reschedule all requeued tasks waiting for retry."""
        pass

    def run(self):
        """Starts the workers main loop."""
        log_wait = lambda: self.logger.info("Waiting for queue...")
        ev_msg_waiting = EventTimer(log_wait, self.empty_msg_emit_every)

        self.pool.run()
        PeriodicWorkController().start()

        # If not running as daemon, and DEBUG logging level is enabled,
        # print pool PIDs and sleep for a second before we start.
        if self.logger.isEnabledFor(logging.DEBUG):
            self.logger.debug("Pool child processes: [%s]" % (
                "|".join(map(str, self.pool.get_worker_pids()))))
            if not self.is_detached:
                time.sleep(1)

        while True:
            try:
                self.execute_next_task()
            except ValueError:
                # execute_next_task didn't return a r/name/id tuple,
                # probably because it got an exception.
                continue
            except EmptyQueue:
                ev_msg_waiting.tick()
                time.sleep(self.queue_wakeup_after)
                continue
            except UnknownTask, exc:
                self.logger.info("Unknown task ignored: %s" % (exc))
                continue
            except Exception, exc:
                self.logger.critical("Message queue raised %s: %s\n%s" % (
                             exc.__class__, exc, traceback.format_exc()))
                continue
<|endoftext|>