<commit_before># Copyright 2014-2020 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

from django.utils.translation import ugettext_lazy as _

from lino.api import rt, dd
from etgen.html import E
from lino.core.diff import ChangeWatcher
from lino_xl.lib.countries.mixins import AddressLocation

class AddressOwner(AddressLocation):
    class Meta:
        abstract = True

    if dd.is_installed('addresses'):

        # def disabled_fields(self, ar):
        #     df = super(AddressOwner, self).disabled_fields(ar)
        #     return df | self.ADDRESS_FIELDS

        def get_address_by_type(self, address_type):
            Address = rt.models.addresses.Address
            try:
                return Address.objects.get(
                    partner=self, address_type=address_type)
            except Address.DoesNotExist:
                return self.get_primary_address()
            except Address.MultipleObjectsReturned:
                return self.get_primary_address()

        def get_primary_address(self):
            if not isinstance(self, dd.plugins.addresses.partner_model):
                return super(AddressOwner,self).get_primary_address()

            Address = rt.models.addresses.Address
            try:
                return Address.objects.get(partner=self, primary=True)
            except Address.DoesNotExist:
                p = self.get_address_parent()
                while p:
                    addr = p.get_primary_address()
                    if addr:
                        return addr
                    p = p.get_address_parent()
            except Address.MultipleObjectsReturned:
                return

        def before_ui_save(self, ar):
            self.sync_to_addresses(ar)
            # self.sync_from_address(self.get_primary_address())
            super(AddressOwner, self).before_ui_save(ar)

        def sync_primary_address(self, request):
            watcher = ChangeWatcher(self)
            self.sync_from_address(self.get_primary_address())
            self.save()
            watcher.send_update(request)

        def sync_to_addresses(self, ar):
            Address = rt.models.addresses.Address
            pa = self.get_primary_address()
            if pa is None:
                values = {}
                for k in Address.ADDRESS_FIELDS:
                    fld = self._meta.get_field(k)
                    v = getattr(self, k)
                    if v != fld.get_default():
                        values[k] = v
                if len(values):
                    addr = Address(partner=self, primary=True, **values)
                    addr.full_clean()
                    addr.save()
                else:
                    for o in obj.addresses_by_partner.filter(primary=True):
                        o.primary = False
                        o.save()

            elif pa != self:
                pa.sync_from_address(self)
                pa.full_clean()
                pa.save()

        def get_overview_elems(self, ar):
            # elems = super(AddressOwner, self).get_overview_elems(ar)
            elems = []
            # e.g. in amici a company inherits from AddressOwner but
            # is not a partner_model because there we want multiple
            # addresses only for persons, not for companies.
            if isinstance(self, dd.plugins.addresses.partner_model):
                sar = ar.spawn('addresses.AddressesByPartner',
                               master_instance=self)
                # btn = sar.as_button(_("Manage addresses"), icon_name="wrench")
                btn = sar.as_button(_("Manage addresses"))
                # elems.append(E.p(btn, align="right"))
                elems.append(E.p(btn))
            return elems
    else:

        def get_overview_elems(self, ar):
            return []

    def get_address_parent(self):
        pass
<commit_msg>fix #3745 cannot create person when addresses is installe<commit_after># Copyright 2014-2020 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

from django.utils.translation import ugettext_lazy as _

from lino.api import rt, dd
from etgen.html import E
from lino.core.diff import ChangeWatcher
from lino_xl.lib.countries.mixins import AddressLocation

class AddressOwner(AddressLocation):
    class Meta:
        abstract = True

    if dd.is_installed('addresses'):

        # def disabled_fields(self, ar):
        #     df = super(AddressOwner, self).disabled_fields(ar)
        #     return df | self.ADDRESS_FIELDS

        def get_address_by_type(self, address_type):
            Address = rt.models.addresses.Address
            try:
                return Address.objects.get(
                    partner=self, address_type=address_type)
            except Address.DoesNotExist:
                return self.get_primary_address()
            except Address.MultipleObjectsReturned:
                return self.get_primary_address()

        def get_primary_address(self):
            if not isinstance(self, dd.plugins.addresses.partner_model):
                return super(AddressOwner,self).get_primary_address()

            Address = rt.models.addresses.Address
            try:
                return Address.objects.get(partner=self, primary=True)
            except Address.DoesNotExist:
                p = self.get_address_parent()
                while p:
                    addr = p.get_primary_address()
                    if addr:
                        return addr
                    p = p.get_address_parent()
            except Address.MultipleObjectsReturned:
                return

        # def before_ui_save(self, ar):
        #     self.sync_to_addresses(ar)
        #     # self.sync_from_address(self.get_primary_address())
        #     super(AddressOwner, self).before_ui_save(ar, cw)

        def after_ui_save(self, ar, cw):
            self.sync_to_addresses(ar)
            # self.sync_from_address(self.get_primary_address())
            super(AddressOwner, self).after_ui_save(ar, cw)

        def sync_primary_address(self, request):
            watcher = ChangeWatcher(self)
            self.sync_from_address(self.get_primary_address())
            self.save()
            watcher.send_update(request)

        def sync_to_addresses(self, ar):
            Address = rt.models.addresses.Address
            pa = self.get_primary_address()
            if pa is None:
                values = {}
                for k in Address.ADDRESS_FIELDS:
                    fld = self._meta.get_field(k)
                    v = getattr(self, k)
                    if v != fld.get_default():
                        values[k] = v
                if len(values):
                    addr = Address(partner=self, primary=True, **values)
                    addr.full_clean()
                    addr.save()
                else:
                    for o in self.addresses_by_partner.filter(primary=True):
                        o.primary = False
                        o.save()

            elif pa != self:
                pa.sync_from_address(self)
                pa.full_clean()
                pa.save()

        def get_overview_elems(self, ar):
            # elems = super(AddressOwner, self).get_overview_elems(ar)
            elems = []
            # e.g. in amici a company inherits from AddressOwner but
            # is not a partner_model because there we want multiple
            # addresses only for persons, not for companies.
            if isinstance(self, dd.plugins.addresses.partner_model):
                sar = ar.spawn('addresses.AddressesByPartner',
                               master_instance=self)
                # btn = sar.as_button(_("Manage addresses"), icon_name="wrench")
                btn = sar.as_button(_("Manage addresses"))
                # elems.append(E.p(btn, align="right"))
                elems.append(E.p(btn))
            return elems
    else:

        def get_overview_elems(self, ar):
            return []

    def get_address_parent(self):
        pass
<|endoftext|>


<commit_before># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Scheduler which supports dependency chaining."""

from twisted.internet import defer
from twisted.python import log
from buildbot import util
from buildbot.db import buildsets
from buildbot.scheduler import Nightly
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers import base, triggerable
from buildbot.status.results import SUCCESS, WARNINGS

import copy
import pending_buildsets

# Monkeypatch this functionality into the buildsets connector.
def BuildsetsConnectorComponentGetBuildsetsForSourceStamp(self, ssid):
  """Retrieve all Buildsets for the given Source Stamp.

  Args:
      ssid: ID of the Source Stamp in question.

  Returns:
      List of Buildset dicts associated with the given Source Stamp.
  """
  def thd(conn):
    bs_tbl = self.db.model.buildsets
    q = bs_tbl.select()
    q = q.where(bs_tbl.c.sourcestampid == ssid)
    res = conn.execute(q)
    return [ self._row2dict(row) for row in res.fetchall() ]
  return self.db.pool.do(thd)
buildsets.BuildsetsConnectorComponent.getBuildsetsForSourceStamp = \
    BuildsetsConnectorComponentGetBuildsetsForSourceStamp

class DependencyFailedError(Exception):
  pass

class DependencyChainScheduler(base.BaseScheduler):
  """Dependent-like scheduler which attempts to satisfy its own dependencies,
  rather than waiting for them to be satisfied."""

  compare_attrs = base.BaseScheduler.compare_attrs + ('dependencies',)

  def __init__(self, dependencies, name, builder_name, properties=None):
    """Initialize this DependencyChainScheduler.

    Args:
        dependencies: List of DependencyChainScheduler instances whose builds
            must be run before this DependencyChainScheduler's builds may run.
        name: Name of this DependencyChainScheduler.
        builder_name: Name of the (single) builder triggered by this Scheduler.
        properties: Optional dictionary of build properties which builds
            triggered by this Scheduler will inherit.
    """
    for dep in dependencies:
      assert isinstance(dep, DependencyChainScheduler)
    if not properties:
      properties = {}
    dependency_list = [d.name for d in dependencies]
    properties['dependencies'] = dependency_list
    base.BaseScheduler.__init__(self, name, [builder_name], properties)
    self.dependencies = dependencies

    self._buildset_addition_subscr = None
    self._buildset_completion_subscr = None
    self.properties.setProperty('dependencies', dependency_list, 'Scheduler')

    # Lock for controlling access to the pending buildsets DB table.
    self._pending_buildset_lock = defer.DeferredLock()

  @util.deferredLocked('_pending_buildset_lock')
  def _unmet_deps_for_source_stamp(self, ssid):
    """Get the unmet dependencies for the given Source Stamp.

    Args:
        ssid: ID of the Source Stamp.

    Returns:
        List of DependencyChainScheduler instances corresponding to the
            dependencies for the given Source Stamp that have not yet been met.

    Raises:
        DependencyFailedError if any of the dependencies have failed.
    """
    def _got_buildset_props(props, buildsets):
      """Callback which runs once the properties for a list of buildsets have
      been retrieved from the database. Determines whether a buildset already
      exists for the given Source Stamp and if not, calls
      _maybe_add_buildset_for_source_stamp to add the buildset or satisfy its
      dependencies.

      Args:
          props: List of tuples of the form: (bool, dict) where each dict
              contains the properties, including the scheduler name, for a
              buildset.
          buildsets: List of buildset dictionaries.
      """
      assert len(buildsets) == len(props)
      unmet_deps = {}
      for dep in self.dependencies:
        unmet_deps[dep.name] = dep
      for i in xrange(len(buildsets)):
        scheduler = props[i][1].get('scheduler', (None, None))[0]
        if (not scheduler) or (scheduler not in unmet_deps):
          continue
        if buildsets[i]['complete']:
          if buildsets[i]['results'] in (SUCCESS, WARNINGS):
            unmet_deps.pop(scheduler)
          else:
            # TODO(borenet): Multiple issues here:
            #   - Are we sure we want to remove "canceled" pending buildsets
            #     from the database? It might be nice to have them present but
            #     with some note that says "canceled because a dep failed."
            #   - We don't correctly propagate failures when a chain of
            #     buildsets depends on a buildset which fails. Only buildsets
            #     which depend directly on the failed buildset get canceled,
            #     and the others remain. This functions correctly in that the
            #     downstream buildsets don't submit build requests, but they
            #     remain in the pending table, without any note about the fact
            #     that they will never run.
            raise DependencyFailedError('Dependency failed: %s' % buildsets[i])
      return [scheduler_instance for
              scheduler_name, scheduler_instance in unmet_deps.items()]

    def _got_buildsets(buildsets):
      """Callback which runs once a list of buildsets have been retrieved from
      the database. Gets the properties for each buildset from the database and
      chains a callback.

      Args:
          buildsets: List of buildset dictionaries.
      """
      dl = []
      for buildset in buildsets:
        dl.append(
            self.master.db.buildsets.getBuildsetProperties(buildset['bsid']))
      d = defer.DeferredList(dl)
      d.addCallback(_got_buildset_props, buildsets)
      return d

    d = self.master.db.buildsets.getBuildsetsForSourceStamp(ssid)
    d.addCallback(_got_buildsets)
    return d

  @util.deferredLocked('_pending_buildset_lock')
  def _cancel_pending_buildsets(self, unused_callback_param, ssid):
    """Cancel all pending Buildsets for the given Source Stamp.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of the Source Stamp whose Buildsets should be aborted.
    """
    return (self.master.skia_db.pending_buildsets
            .cancel_pending_buildsets(ssid, self.name))

  @util.deferredLocked('_pending_buildset_lock')
  def _actually_add_buildset_for_source_stamp(self, ssid):
    """Unconditionally add Buildsets for the given Source Stamp, removing the
    associated pending Buildsets from the pending Buildsets table.

    Args:
        ssid: ID of the Source Stamp to build.
    """
    def _got_pending(pending):
      dl = []
      for bs in pending:
        dl.append(
            base.BaseScheduler.addBuildsetForSourceStamp(self,
                ssid=ssid,
                reason=bs['reason'],
                external_idstring=bs['external_idstring'],
                properties=bs['properties']))
      d = defer.DeferredList(dl)
      d.addCallback(self.master.skia_db.pending_buildsets
                        .cancel_pending_buildsets(ssid, self.name))
      return d
    d = self.master.skia_db.pending_buildsets.get_pending_buildsets(ssid,
                                                                    self.name)
    d.addCallback(_got_pending)
    return d

  def _maybe_add_buildset_for_source_stamp(self, unused_callback_param, ssid,
                                           launch_dependencies, arguments):
    """If all dependencies have been met for the given Source Stamp, add a build
    sets for each pending build for the given Source Stamp. If not, and if it
    was requested, launch the unmet dependencies for the given Source Stamp.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of a Source Stamp.
        launch_dependencies: Boolean; whether or not the dependencies for this
            DependencyChainScheduler should be triggered if they haven't yet
            been satisfied.
        arguments: Dictionary of arguments to be passed to
            addBuildsetForSourceStamp.
    """
    def _got_dependencies(unmet_dependencies):
      if unmet_dependencies:
        if launch_dependencies:
          dl = [dep.addBuildsetForSourceStamp(ssid, **arguments)
                for dep in unmet_dependencies]
          return defer.DeferredList(dl)
      else:
        return self._actually_add_buildset_for_source_stamp(ssid=ssid)
    d = self._unmet_deps_for_source_stamp(ssid)
    d.addCallback(_got_dependencies)
    d.addErrback(self._cancel_pending_buildsets, ssid)
    return d

  @util.deferredLocked('_pending_buildset_lock')
  def _maybe_add_pending_buildset_for_source_stamp(self, unused_callback_param,
                                                   ssid, arguments):
    """Add a Buildset to the pending Buildsets table if no pending or already-
    inserted Buildset exists.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of the Source Stamp.
        arguments: Dictionary of arguments to be passed to addBuildset.
    """
    def _got_pending_buildsets(pending):
      """Callback which runs once the list of pending Buildsets has been
      retrieved from the database. If there is no pending Buildset with an
      identical set of arguments, add a pending Buildset.

      Args:
          pending: List of dicts of Buildset arguments representing not-yet-
              inserted Buildsets.
      """
      for bs in pending:
        if (str(bs['reason']) == str(arguments.get('reason', '')) and
            bs['properties'] == arguments.get('properties', {})):
          return defer.succeed(None)
      d = (self.master.skia_db.pending_buildsets
           .add_pending_buildset(ssid, self.name,
                                 [d.name for d in self.dependencies],
                                 **arguments))
      return d

    def _got_buildset_props(props, buildsets):
      """Callback which runs once the properties for a list of buildsets have
      been retrieved from the database. Determines whether a buildset already
      exists for the given Source Stamp and if not, calls
      _maybe_add_buildset_for_source_stamp to add the buildset or satisfy its
      dependencies.

      Args:
          props: List of tuples of the form: (bool, dict) where each dict
              contains the properties, including the scheduler name, for a
              buildset.
          buildsets: List of buildset dictionaries.
      """
      assert len(buildsets) == len(props)
      for i in xrange(len(buildsets)):
        if props[i][1].get('scheduler', (None, None))[0] == self.name:
          # If there's already a Buildset, don't insert one.
          return defer.succeed(None)
      d = (self.master.skia_db.pending_buildsets
           .get_pending_buildsets(ssid, self.name))
      d.addCallback(_got_pending_buildsets)
      return d

    def _got_buildsets(buildsets):
      """Callback which runs once a list of buildsets have been retrieved from
      the database. Gets the properties for each buildset from the database and
      chains a callback.

      Args:
          buildsets: List of buildset dictionaries.
      """
      dl = []
      for buildset in buildsets:
        dl.append(
            self.master.db.buildsets.getBuildsetProperties(buildset['bsid']))
      d = defer.DeferredList(dl)
      d.addCallback(_got_buildset_props, buildsets)
      return d
    d = self.master.db.buildsets.getBuildsetsForSourceStamp(ssid)
    d.addCallback(_got_buildsets)
    return d

  def addBuildsetForSourceStamp(self, ssid, **arguments):
    """Attempt to add a buildset for the given Source Stamp.

    For other Schedulers this function just adds the buildset. Instead, this
    Scheduler adds an entry to the pending dictionary and then calls
    _maybe_add_buildset_for_source_stamp, which will launch any unmet
    dependencies or add the buildset.

    Args:
        ssid: ID of a Source Stamp.
        arguments: Dictionary of arguments to be passed to
            addBuildsetForSourceStamp.
    """
    d = self._maybe_add_pending_buildset_for_source_stamp(
        unused_callback_param=None, ssid=ssid, arguments=arguments)
    d.addCallback(self._maybe_add_buildset_for_source_stamp, ssid=ssid,
                  launch_dependencies=True, arguments=arguments)
    return d

  def startService(self):
    """Called when the DependencyChainScheduler starts running. Registers
    callback function for when buildsets are completed."""
    self._buildset_completion_subscr = \
        self.master.subscribeToBuildsetCompletions(
            self._check_completed_buildsets)
    # check for any buildsets completed before we started
    d = self._check_completed_buildsets(None, None)
    d.addErrback(log.err, 'while checking for completed buildsets in start')

  def stopService(self):
    """Called when the DependencyChainScheduler stops running. Unregisters all
    callback functions."""
    if self._buildset_completion_subscr:
      self._buildset_completion_subscr.unsubscribe()
    return defer.succeed(None)

  def _buildset_completed(self, bsid, result):
    """Callback function which runs when any buildset is completed on the
    current build master. Just runs the _check_completed_buildsets function to
    determine whether any action is necessary.

    Args:
        bsid: ID of the completed buildset.
        result: Result of the completed buildset. Possible values are defined in
            builbot.status.results.
    """
    d = self._check_completed_buildsets(bsid, result)
    d.addErrback(log.err, 'while checking for completed buildsets')

  def _check_completed_buildsets(self, bsid, result):
    """For each newly-completed buildset, determine whether the buildset
    satisfied a dependency for a Source Stamp. If so, run
    _maybe_add_buildset_for_source_stamp which will add a buildset for the
    source stamp if all of its dependencies have been satisfied.

    Args:
        bsid: ID of the completed buildset.
        result: Result of the completed buildset. Possible values are defined in
            builbot.status.results.
    """

    def _got_buildset(buildset):
      if buildset:
        return self._maybe_add_buildset_for_source_stamp(
            unused_callback_param=None,
            ssid=buildset['sourcestampid'],
            launch_dependencies=False,
            arguments=None)
      else:
        return defer.succeed(None)

    d = self.master.db.buildsets.getBuildset(bsid)
    d.addCallback(_got_buildset)
    return d

class PerCommitScheduler(SingleBranchScheduler):
  """Scheduler which triggers a set of schedulers on every commit."""
  def __init__(self, dependencies, *args, **kwargs):
    """Initialize the PerCommitScheduler.

    Args:
        dependencies: List of schedulers which should trigger on every commit.
    """
    SingleBranchScheduler.__init__(self, builderNames=[], *args, **kwargs)
    self._dependencies = dependencies

  def addBuildsetForSourceStamp(self, *args, **kwargs):
    """Override of SingleBranchScheduler.addBuildsetForSourceStamp. Rather than
    actually adding a Buildset, trigger another set of schedulers."""
    dl = []
    for dependency in self._dependencies:
      dl.append(dependency.addBuildsetForSourceStamp(*args, **kwargs))
    return defer.DeferredList(dl)

class NightlyScheduler(Nightly):
  """Scheduler which triggers a set of schedulers every night."""
  def __init__(self, dependencies, *args, **kwargs):
    """Initialize the NightlyScheduler.

    Args:
        dependencies: List of schedulers which should trigger every night.
    """
    Nightly.__init__(self, builderNames=[], *args, **kwargs)
    self._dependencies = dependencies

  def addBuildsetForSourceStamp(self, *args, **kwargs):
    """Override of Nightly.addBuildsetForSourceStamp. Rather than actually
    adding a Buildset, trigger another set of schedulers."""
    dl = []
    for dependency in self._dependencies:
      dl.append(dependency.addBuildsetForSourceStamp(*args, **kwargs))
    return defer.DeferredList(dl)

<commit_msg>Tweaks to DependencyChainScheduler<commit_after># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Scheduler which supports dependency chaining."""

from twisted.internet import defer
from twisted.python import log
from buildbot import util
from buildbot.db import buildsets
from buildbot.process import properties
from buildbot.scheduler import Nightly
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers import base, triggerable
from buildbot.status.results import SUCCESS, WARNINGS

import copy
import pending_buildsets

# Monkeypatch this functionality into the buildsets connector.
def BuildsetsConnectorComponentGetBuildsetsForSourceStamp(self, ssid):
  """Retrieve all Buildsets for the given Source Stamp.

  Args:
      ssid: ID of the Source Stamp in question.

  Returns:
      List of Buildset dicts associated with the given Source Stamp.
  """
  def thd(conn):
    bs_tbl = self.db.model.buildsets
    q = bs_tbl.select()
    q = q.where(bs_tbl.c.sourcestampid == ssid)
    res = conn.execute(q)
    return [ self._row2dict(row) for row in res.fetchall() ]
  return self.db.pool.do(thd)
buildsets.BuildsetsConnectorComponent.getBuildsetsForSourceStamp = \
    BuildsetsConnectorComponentGetBuildsetsForSourceStamp

class DependencyFailedError(Exception):
  pass

class DependencyChainScheduler(base.BaseScheduler):
  """Dependent-like scheduler which attempts to satisfy its own dependencies,
  rather than waiting for them to be satisfied."""

  compare_attrs = base.BaseScheduler.compare_attrs + ('dependencies',)
  props_from_dependencies = ['got_revision']

  def __init__(self, dependencies, name, builder_name, properties=None):
    """Initialize this DependencyChainScheduler.

    Args:
        dependencies: List of DependencyChainScheduler instances whose builds
            must be run before this DependencyChainScheduler's builds may run.
        name: Name of this DependencyChainScheduler.
        builder_name: Name of the (single) builder triggered by this Scheduler.
        properties: Optional dictionary of build properties which builds
            triggered by this Scheduler will inherit.
    """
    for dep in dependencies:
      assert isinstance(dep, DependencyChainScheduler)
    if not properties:
      properties = {}
    dependency_list = [d.name for d in dependencies]
    properties['dependencies'] = dependency_list
    base.BaseScheduler.__init__(self, name, [builder_name], properties)
    self.dependencies = dependencies

    self._buildset_addition_subscr = None
    self._buildset_completion_subscr = None
    self.properties.setProperty('dependencies', dependency_list, 'Scheduler')

    # Lock for controlling access to the pending buildsets DB table.
    self._pending_buildset_lock = defer.DeferredLock()

  @util.deferredLocked('_pending_buildset_lock')
  def _unmet_deps_for_source_stamp(self, ssid):
    """Get the unmet dependencies for the given Source Stamp.

    Args:
        ssid: ID of the Source Stamp.

    Returns:
        List of DependencyChainScheduler instances corresponding to the
            dependencies for the given Source Stamp that have not yet been met.

    Raises:
        DependencyFailedError if any of the dependencies have failed.
    """
    def _got_buildset_props(props, buildsets):
      """Callback which runs once the properties for a list of buildsets have
      been retrieved from the database. Determines whether a buildset already
      exists for the given Source Stamp and if not, calls
      _maybe_add_buildset_for_source_stamp to add the buildset or satisfy its
      dependencies.

      Args:
          props: List of tuples of the form: (bool, dict) where each dict
              contains the properties, including the scheduler name, for a
              buildset.
          buildsets: List of buildset dictionaries.
      """
      assert len(buildsets) == len(props)
      unmet_deps = {}
      for dep in self.dependencies:
        unmet_deps[dep.name] = dep
      for i in xrange(len(buildsets)):
        scheduler = props[i][1].get('scheduler', (None, None))[0]
        if (not scheduler) or (scheduler not in unmet_deps):
          continue
        if buildsets[i]['complete']:
          if buildsets[i]['results'] in (SUCCESS, WARNINGS):
            unmet_deps.pop(scheduler)
          else:
            raise DependencyFailedError('Dependency failed: %s' % buildsets[i])
      return [scheduler_instance for
              scheduler_name, scheduler_instance in unmet_deps.items()]

    def _got_buildsets(buildsets):
      """Callback which runs once a list of buildsets have been retrieved from
      the database. Gets the properties for each buildset from the database and
      chains a callback.

      Args:
          buildsets: List of buildset dictionaries.
      """
      dl = []
      for buildset in buildsets:
        dl.append(
            self.master.db.buildsets.getBuildsetProperties(buildset['bsid']))
      d = defer.DeferredList(dl)
      d.addCallback(_got_buildset_props, buildsets)
      return d

    d = self.master.db.buildsets.getBuildsetsForSourceStamp(ssid)
    d.addCallback(_got_buildsets)
    return d

  @util.deferredLocked('_pending_buildset_lock')
  def _cancel_pending_buildsets(self, unused_callback_param, ssid):
    """Cancel all pending Buildsets for the given Source Stamp.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of the Source Stamp whose Buildsets should be aborted.
    """
    return (self.master.skia_db.pending_buildsets
            .cancel_pending_buildsets(ssid, self.name))

  @util.deferredLocked('_pending_buildset_lock')
  def _actually_add_buildset_for_source_stamp(self, ssid, build_props):
    """Unconditionally add Buildsets for the given Source Stamp, removing the
    associated pending Buildsets from the pending Buildsets table.

    Args:
        ssid: ID of the Source Stamp to build.
        build_props: instance of properties.Properties; properties to add to the
           build.
    """
    def _added_buildsets(_unused_callback_result):
      return (self.master.skia_db.pending_buildsets
                  .cancel_pending_buildsets(ssid, self.name))

    def _got_pending(pending):
      dl = []
      for bs in pending:
        props = properties.Properties()
        props.updateFromProperties(build_props)
        props.update({'ssid': ssid}, 'Scheduler')
        props.update(bs['properties'] or {}, 'Scheduler')
        dl.append(
            base.BaseScheduler.addBuildsetForSourceStamp(self,
                ssid=ssid,
                reason=bs['reason'],
                external_idstring=bs['external_idstring'],
                properties=props))
      if not dl:
        return defer.succeed(None)
      d = defer.DeferredList(dl)
      d.addCallback(_added_buildsets)
      return d
    d = self.master.skia_db.pending_buildsets.get_pending_buildsets(ssid,
                                                                    self.name)
    d.addCallback(_got_pending)
    return d

  def _maybe_add_buildset_for_source_stamp(self, unused_callback_param, ssid,
                                           launch_dependencies, arguments,
                                           set_properties=None):
    """If all dependencies have been met for the given Source Stamp, add a build
    sets for each pending build for the given Source Stamp. If not, and if it
    was requested, launch the unmet dependencies for the given Source Stamp.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of a Source Stamp.
        launch_dependencies: Boolean; whether or not the dependencies for this
            DependencyChainScheduler should be triggered if they haven't yet
            been satisfied.
        arguments: Dictionary of arguments to be passed to
            addBuildsetForSourceStamp.
        set_properties: optional dictionary of build properties to be propagated
            from a dependency to the current buildset.
    """
    def _got_dependencies(unmet_dependencies):
      if unmet_dependencies:
        if launch_dependencies:
          dl = [dep.addBuildsetForSourceStamp(ssid, **arguments)
                for dep in unmet_dependencies]
          return defer.DeferredList(dl)
      else:
        # TODO(borenet): Actually get build properties from the dependencies.
        props = properties.Properties()
        if set_properties:
          props.update(set_properties, 'Dependency')
        return self._actually_add_buildset_for_source_stamp(ssid=ssid,
                                                            build_props=props)
    d = self._unmet_deps_for_source_stamp(ssid)
    d.addCallback(_got_dependencies)
    return d

  @util.deferredLocked('_pending_buildset_lock')
  def _maybe_add_pending_buildset_for_source_stamp(self, unused_callback_param,
                                                   ssid, arguments):
    """Add a Buildset to the pending Buildsets table if no pending or already-
    inserted Buildset exists.

    Args:
        unused_callback_param: Unused; placeholder for the return value of a
            function to which this function may be attached as a callback.
        ssid: ID of the Source Stamp.
        arguments: Dictionary of arguments to be passed to addBuildset.
    """
    def _got_pending_buildsets(pending):
      """Callback which runs once the list of pending Buildsets has been
      retrieved from the database. If there is no pending Buildset with an
      identical set of arguments, add a pending Buildset.

      Args:
          pending: List of dicts of Buildset arguments representing not-yet-
              inserted Buildsets.
      """
      for bs in pending:
        if (str(bs['reason']) == str(arguments.get('reason', '')) and
            bs['properties'] == arguments.get('properties', {})):
          return defer.succeed(None)
      d = (self.master.skia_db.pending_buildsets
           .add_pending_buildset(ssid, self.name,
                                 [d.name for d in self.dependencies],
                                 **arguments))
      return d

    def _got_buildset_props(props, buildsets):
      """Callback which runs once the properties for a list of buildsets have
      been retrieved from the database. Determines whether a buildset already
      exists for the given Source Stamp and if not, calls
      _maybe_add_buildset_for_source_stamp to add the buildset or satisfy its
      dependencies.

      Args:
          props: List of tuples of the form: (bool, dict) where each dict
              contains the properties, including the scheduler name, for a
              buildset.
          buildsets: List of buildset dictionaries.
      """
      assert len(buildsets) == len(props)
      for i in xrange(len(buildsets)):
        if props[i][1].get('scheduler', (None, None))[0] == self.name:
          # If there's already a Buildset, don't insert one.
          return defer.succeed(None)
      d = (self.master.skia_db.pending_buildsets
           .get_pending_buildsets(ssid, self.name))
      d.addCallback(_got_pending_buildsets)
      return d

    def _got_buildsets(buildsets):
      """Callback which runs once a list of buildsets have been retrieved from
      the database. Gets the properties for each buildset from the database and
      chains a callback.

      Args:
          buildsets: List of buildset dictionaries.
      """
      dl = []
      for buildset in buildsets:
        dl.append(
            self.master.db.buildsets.getBuildsetProperties(buildset['bsid']))
      d = defer.DeferredList(dl)
      d.addCallback(_got_buildset_props, buildsets)
      return d
    d = self.master.db.buildsets.getBuildsetsForSourceStamp(ssid)
    d.addCallback(_got_buildsets)
    return d

  def addBuildsetForSourceStamp(self, ssid, **arguments):
    """Attempt to add a buildset for the given Source Stamp.

    For other Schedulers this function just adds the buildset. Instead, this
    Scheduler adds an entry to the pending dictionary and then calls
    _maybe_add_buildset_for_source_stamp, which will launch any unmet
    dependencies or add the buildset.

    Args:
        ssid: ID of a Source Stamp.
        arguments: Dictionary of arguments to be passed to
            addBuildsetForSourceStamp.
    """
    d = self._maybe_add_pending_buildset_for_source_stamp(
        unused_callback_param=None, ssid=ssid, arguments=arguments)
    d.addCallback(self._maybe_add_buildset_for_source_stamp, ssid=ssid,
                  launch_dependencies=True, arguments=arguments)
    return d

  def startService(self):
    """Called when the DependencyChainScheduler starts running. Registers
    callback function for when buildsets are completed."""
    self._buildset_completion_subscr = \
        self.master.subscribeToBuildsetCompletions(
            self._check_completed_buildsets)
    # check for any buildsets completed before we started
    d = self._check_completed_buildsets(None, None)
    d.addErrback(log.err, 'while checking for completed buildsets in start')

  def stopService(self):
    """Called when the DependencyChainScheduler stops running. Unregisters all
    callback functions."""
    if self._buildset_completion_subscr:
      self._buildset_completion_subscr.unsubscribe()
    return defer.succeed(None)

  def _buildset_completed(self, bsid, result):
    """Callback function which runs when any buildset is completed on the
    current build master. Just runs the _check_completed_buildsets function to
    determine whether any action is necessary.

    Args:
        bsid: ID of the completed buildset.
        result: Result of the completed buildset. Possible values are defined in
            builbot.status.results.
    """
    d = self._check_completed_buildsets(bsid, result)
    d.addErrback(log.err, 'while checking for completed buildsets')

  def _check_completed_buildsets(self, bsid, result):
    """For each newly-completed buildset, determine whether the buildset
    satisfied a dependency for a Source Stamp. If so, run
    _maybe_add_buildset_for_source_stamp which will add a buildset for the
    source stamp if all of its dependencies have been satisfied.

    Args:
        bsid: ID of the completed buildset.
        result: Result of the completed buildset. Possible values are defined in
            builbot.status.results.
    """

    def _got_buildset_props(props, buildset):
      properties_to_pass = {}
      if props:
        for k, v in props.iteritems():
          if k in DependencyChainScheduler.props_from_dependencies:
            properties_to_pass[k] = v[0]
      return self._maybe_add_buildset_for_source_stamp(
            unused_callback_param=None,
            ssid=buildset['sourcestampid'],
            launch_dependencies=False,
            arguments=None,
            set_properties=properties_to_pass)

    def _got_buildset(buildset):
      if buildset:
        d = self.master.db.buildsets.getBuildsetProperties(buildset['bsid'])
        d.addCallback(_got_buildset_props, buildset)
        return d
      else:
        return defer.succeed(None)

    d = self.master.db.buildsets.getBuildset(bsid)
    d.addCallback(_got_buildset)
    return d

class PerCommitScheduler(SingleBranchScheduler):
  """Scheduler which triggers a set of schedulers on every commit."""
  def __init__(self, dependencies, *args, **kwargs):
    """Initialize the PerCommitScheduler.

    Args:
        dependencies: List of schedulers which should trigger on every commit.
    """
    SingleBranchScheduler.__init__(self, builderNames=[], *args, **kwargs)
    self._dependencies = dependencies

  def addBuildsetForSourceStamp(self, *args, **kwargs):
    """Override of SingleBranchScheduler.addBuildsetForSourceStamp. Rather than
    actually adding a Buildset, trigger another set of schedulers."""
    dl = []
    for dependency in self._dependencies:
      dl.append(dependency.addBuildsetForSourceStamp(*args, **kwargs))
    return defer.DeferredList(dl)

class NightlyScheduler(Nightly):
  """Scheduler which triggers a set of schedulers every night."""
  def __init__(self, dependencies, *args, **kwargs):
    """Initialize the NightlyScheduler.

    Args:
        dependencies: List of schedulers which should trigger every night.
    """
    Nightly.__init__(self, builderNames=[], *args, **kwargs)
    self._dependencies = dependencies

  def addBuildsetForSourceStamp(self, *args, **kwargs):
    """Override of Nightly.addBuildsetForSourceStamp. Rather than actually
    adding a Buildset, trigger another set of schedulers."""
    dl = []
    for dependency in self._dependencies:
      dl.append(dependency.addBuildsetForSourceStamp(*args, **kwargs))
    return defer.DeferredList(dl)

<|endoftext|>


<commit_before>'''
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
'''

from arakoon.ArakoonExceptions import *
from system_tests_common import *
from system_tests_anomaly import *
from system_tests_basic import *

import arakoon

AM_WORK_LIST_MAX_ITEMS = 10
AM_MAX_WORKITEM_OPS = 100000

def dummy():
    pass

monkey_catalogue = [
    ( set_get_and_delete, True ) ,
    ( prefix_scenario, False ) ,
    ( range_scenario, False ) ,
    ( range_entries_scenario, False ) ,
    #( tes_and_set_scenario, False )
    ]

network_error_regexes = [
    ArakoonSockReadNoBytes._msg,
    ArakoonSockNotReadable._msg,
    ArakoonSockRecvError._msg,
    ArakoonSockRecvClosed._msg,
    ArakoonSockSendError._msg
    ]

restart_error_regexes = [
    ArakoonNoMaster._msg ,
    ArakoonNodeNotMaster._msg
    ]

restart_error_regexes.extend( network_error_regexes )

monkey_disruptions_catalogue = [
#    ( delayed_restart_1st_node, restart_error_regexes ) ,
#    ( delayed_restart_2nd_node, restart_error_regexes ) ,
#    ( delayed_restart_3rd_node, restart_error_regexes ) ,
#    ( delayed_restart_all_nodes, restart_error_regexes ) ,
#    ( iterate_block_unblock_single_slave, restart_error_regexes ) ,
#    ( iterate_block_unblock_both_slaves, restart_error_regexes ) ,
#    ( iterate_block_unblock_master, restart_error_regexes ) ,
    ( dummy, [] )
    ]

<commit_msg>system_tests_basic moved<commit_after>'''
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
'''

from arakoon.ArakoonExceptions import *
from system_tests_common import *
from system_tests_anomaly import *
from quick.system_tests_basic import *

import arakoon

AM_WORK_LIST_MAX_ITEMS = 10
AM_MAX_WORKITEM_OPS = 100000

def dummy():
    pass

monkey_catalogue = [
    ( set_get_and_delete, True ) ,
    ( prefix_scenario, False ) ,
    ( range_scenario, False ) ,
    ( range_entries_scenario, False ) ,
    #( tes_and_set_scenario, False )
    ]

network_error_regexes = [
    ArakoonSockReadNoBytes._msg,
    ArakoonSockNotReadable._msg,
    ArakoonSockRecvError._msg,
    ArakoonSockRecvClosed._msg,
    ArakoonSockSendError._msg
    ]

restart_error_regexes = [
    ArakoonNoMaster._msg ,
    ArakoonNodeNotMaster._msg
    ]

restart_error_regexes.extend( network_error_regexes )

monkey_disruptions_catalogue = [
#    ( delayed_restart_1st_node, restart_error_regexes ) ,
#    ( delayed_restart_2nd_node, restart_error_regexes ) ,
#    ( delayed_restart_3rd_node, restart_error_regexes ) ,
#    ( delayed_restart_all_nodes, restart_error_regexes ) ,
#    ( iterate_block_unblock_single_slave, restart_error_regexes ) ,
#    ( iterate_block_unblock_both_slaves, restart_error_regexes ) ,
#    ( iterate_block_unblock_master, restart_error_regexes ) ,
    ( dummy, [] )
    ]

<|endoftext|>


<commit_before>import logging

from pywps import ComplexInput, ComplexOutput
from pywps import Format
from pywps import LiteralInput
from pywps import Process
from pywps.app.Common import Metadata

from eggshell.plot import plt_ncdata
from eggshell.utils import extract_archive
from eggshell.nc.nc_utils import get_variable
# from eggshell.utils import rename_complexinputs
# from eggshell.log import init_process_logger

LOGGER = logging.getLogger("PYWPS")

class PlotuncertaintyProcess(Process):
    def __init__(self):
        inputs = [
            ComplexInput('resource', 'Resource',
                         abstract='NetCDF Files (with one variable) or archive (tar/zip) containing NetCDF files.',
                         metadata=[Metadata('Info')],
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            LiteralInput("variable", "Variable",
                         abstract="Variable to be expected in the input files (variable will be detected if not set)",
                         default=None,
                         data_type='string',
                         min_occurs=0,
                         max_occurs=1,
                         ),
        ]

        outputs = [
            # ComplexOutput('output_log', 'Logging information',
            #               abstract="Collected logs during process run.",
            #               as_reference=True,
            #               supported_formats=[Format('text/plain')]
            #               ),
            #
            # ComplexOutput("plotout_spaghetti", "Visualisation, Spaghetti plot",
            #               abstract="Visualisation of single variables as a spaghetti plot",
            #               supported_formats=[Format("image/png")],
            #               as_reference=True,
            #               ),

            ComplexOutput("plotout_uncertainty", "Visualisation Uncertainty plot",
                          abstract="Visualisation of single variables ensemble running mean with uncertainty",
                          supported_formats=[Format("image/png")],
                          as_reference=True,
                          )
            # TODO: include window for running mean
        ]

        super(PlotuncertaintyProcess, self).__init__(
            self._handler,
            identifier="plot_uncertainty",
            title="Timeseries as uncertainty plot",
            version="0.1",
            metadata=[
                Metadata('Doc',
                         'https://flyingpigeon.readthedocs.io/en/latest/processes_des.html#data-visualization'),
            ],
            abstract="Outputs timeseries of all inputfiles with the ensemble running mean and correponding bandwidth.\
                      The single lines are colorcoded according to the IPCC graphic guidelines.",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        # init_process_logger('log.txt')
        # response.outputs['output_log'].file = 'log.txt'

        ncfiles = extract_archive(
            resources=[inpt.file for inpt in request.inputs['resource']],
            dir_output=self.workdir)

        if 'variable' in request.inputs:
            var = request.inputs['variable'][0].data
        else:
            var = get_variable(ncfiles[0])
            #  var = ncfiles[0].split("_")[0]

        response.update_status('plotting variable {}'.format(var), 10)

        try:
            plotout_file = plt_ncdata.uncertainty(ncfiles,
                                                  variable=var,
                                                  title='Field mean of {}'.format(var),
                                                  dir_output=self.workdir,
                                                  )
            LOGGER.info("uncertainty plot done")
            response.update_status('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 50)
            response.outputs['plotout_uncertainty'].file = plotout_file
        except Exception as e:
            raise Exception("Uncertainty plot failed : {}".format(e))

        response.update_status('visualisation done', 100)
        return response
<commit_msg>improve plot functions uncertainty<commit_after>import logging

from pywps import ComplexInput, ComplexOutput
from pywps import Format
from pywps import LiteralInput
from pywps import Process
from pywps.app.Common import Metadata

from eggshell.plot import plt_ncdata
from eggshell.utils import extract_archive
from eggshell.nc.nc_utils import get_variable
# from eggshell.utils import rename_complexinputs
# from eggshell.log import init_process_logger

LOGGER = logging.getLogger("PYWPS")

class PlotuncertaintyProcess(Process):
    def __init__(self):
        inputs = [
            ComplexInput('resource', 'Resource',
                         abstract='NetCDF Files (with one variable) or archive (tar/zip) containing NetCDF files.',
                         metadata=[Metadata('Info')],
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            LiteralInput("variable", "Variable",
                         abstract="Variable to be expected in the input files (variable will be detected if not set)",
                         default=None,
                         data_type='string',
                         min_occurs=0,
                         max_occurs=1,
                         ),

            LiteralInput("title", "Title",
                         abstract="Title to be written over the graphic",
                         default=None,
                         data_type='string',
                         min_occurs=0,
                         max_occurs=1,
                         ),

            LiteralInput('delta', 'Delta',
                         abstract='To set an offset for the values.'\
                                   'e.g. -273.15 to transform Kelvin to Celsius',
                         data_type='float',
                         default=0,
                         ),

            LiteralInput('figsize', 'Figure Size',   data_type='string',
                         abstract='Give two numbers (as a string:"7,10") to define the size of the graphic',
                         default='10,10',
                         ),

        ]

        outputs = [
            # ComplexOutput('output_log', 'Logging information',
            #               abstract="Collected logs during process run.",
            #               as_reference=True,
            #               supported_formats=[Format('text/plain')]
            #               ),
            #
            # ComplexOutput("plotout_spaghetti", "Visualisation, Spaghetti plot",
            #               abstract="Visualisation of single variables as a spaghetti plot",
            #               supported_formats=[Format("image/png")],
            #               as_reference=True,
            #               ),

            ComplexOutput("plotout_uncertainty", "Visualisation Uncertainty plot",
                          abstract="Visualisation of single variables ensemble running mean with uncertainty",
                          supported_formats=[Format("image/png")],
                          as_reference=True,
                          )
            # TODO: include window for running mean
        ]

        super(PlotuncertaintyProcess, self).__init__(
            self._handler,
            identifier="plot_uncertainty",
            title="Timeseries as uncertainty plot",
            version="0.1",
            metadata=[
                Metadata('Doc',
                         'https://flyingpigeon.readthedocs.io/en/latest/processes_des.html#data-visualization'),
            ],
            abstract="Outputs timeseries of all inputfiles with the ensemble running mean and correponding bandwidth.\
                      The single lines are colorcoded according to the IPCC graphic guidelines.",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        # init_process_logger('log.txt')
        # response.outputs['output_log'].file = 'log.txt'

        ncfiles = extract_archive(
            resources=[inpt.file for inpt in request.inputs['resource']],
            dir_output=self.workdir)

        if 'variable' in request.inputs:
            var = request.inputs['variable'][0].data
        else:
            var = get_variable(ncfiles[0])
            #  var = ncfiles[0].split("_")[0]
        if 'delta' in request.inputs:
            delta = request.inputs['delta'][0].data
        else:
            delta = 0
        if 'title' in request.inputs:
            title = request.inputs['title'][0].data
        else:
            title = None

        f_str = request.inputs['figsize'][0].data
        figsize = tuple([float(f) for f in f_str.split(',')])
        response.update_status('plotting variable {}'.format(var), 10)

        try:
            plotout_file = plt_ncdata.uncertainty(ncfiles,
                                                  variable=var,
                                                  title=title,
                                                  delta=delta,
                                                  figsize=figsize,
                                                  dir_output=self.workdir,
                                                  )
            LOGGER.info("uncertainty plot done")
            response.update_status('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 50)
            response.outputs['plotout_uncertainty'].file = plotout_file
        except Exception as e:
            raise Exception("Uncertainty plot failed : {}".format(e))

        response.update_status('visualisation done', 100)
        return response
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Jun 01 19:47:26 2015

@author: sherlockye
"""

import re
import sys
pinyin_dict = {}

def LoadPinyinDict():
    global pinyin_dict
    for line in open('pinyin.txt'):
        line = line.replace('\n','')
        line = line.split(":")
        pinyin_dict[unicode(line[0],"utf-8")] = line[1].split(",")

def UniformPinyin(pinyin):
    pinyin, number = re.subn('(.)h(.*)', r'\1\2', pinyin)
    pinyin, number = re.subn('(.*)ng', r'\1n', pinyin)
    return pinyin

def GetPinyin(word):
    if pinyin_dict.has_key(word):
        return pinyin_dict[word][0]
    else:
        return "-"

def convertPinyin(word):
    # word = word.replace(",","")
    return map(lambda x:UniformPinyin(GetPinyin(x)),word)

def ValidCheck(wordList_py,dstWord_py):
    idx = []
    for wordidx,wordpy in enumerate(wordList_py):
        inflag = False
        for pyidx,py in enumerate(wordpy):
            if py in dstWord_py:
                dstidx = dstWord_py.index(py)
                idx.append([wordidx,pyidx,dstidx])
                inflag = True
        if not inflag:
            return []
    return idx

def PrintValidResult(pinyinSet,oriText,wordList):
    line = oriText
    try:
        dstWord_py =  convertPinyin(line)
    except:
        print "Error:%s"%(oriText)
        return
    ReplaceRule = ValidCheck(pinyinSet,dstWord_py)
    if len(ReplaceRule):
        result = list(line)
        for replace_idx in ReplaceRule:
            dst = wordList[replace_idx[0]][replace_idx[1]]
            src = result[replace_idx[2]]
            line = line.replace(src,dst)
        print line,'<<----',oriText

def ReadGushi(filename,pinyinSet,wordList):
    for (linenum,oriText) in enumerate(open(filename)):
        oriText = unicode(oriText.replace('\n',''),"utf-8")
        regexp = "<<(.*)>>:(.*)$"
        result = re.findall(regexp,oriText)
        for (title,poet) in result:
            poet = poet.replace(unicode('，',"utf-8"),'---')
            poet = poet.replace(unicode('。',"utf-8"),'---')
            poet = poet.replace(unicode('？',"utf-8"),'---')
            poet = poet.replace(unicode('、',"utf-8"),'---')
            poet = poet.replace(unicode('！',"utf-8"),'---')
            poet = poet.replace(unicode('；',"utf-8"),'---')
            poet = poet.replace(unicode('：',"utf-8"),'---')
            poet = poet.replace(unicode(':',"utf-8"),'---')
            poet = poet.replace(unicode('”',"utf-8"),'---')
            poet = poet.replace(unicode('“',"utf-8"),'---')
            poet = poet.split('---')
            for p in poet:
                PrintValidResult(pinyinSet,p,wordList)

def main(argv1,argv2):
    wordList = [argv1,argv2]
    wordList = map(lambda x:unicode(x,"utf-8"),wordList)
    pinyinSet = map(lambda x:convertPinyin(x),wordList)
    # ReadGushi("tangshi.txt",pinyinSet,wordList)
    #ReadGushi("songci.txt",pinyinSet,wordList)
    for (linenum,oriText) in enumerate(open('chengyu.txt')):
        oriText = unicode(oriText.replace('\n',''),"utf-8")
        PrintValidResult(pinyinSet,oriText,wordList)

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print "python InspirationHub.py input1 input2"
    else:
        LoadPinyinDict()
        main(sys.argv[1],sys.argv[2])<commit_msg>windows默认GB2312编码<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Jun 01 19:47:26 2015

@author: sherlockye
"""

import re
import sys
pinyin_dict = {}

def LoadPinyinDict():
    global pinyin_dict
    for line in open('pinyin.txt'):
        line = line.replace('\n','')
        line = line.split(":")
        pinyin_dict[unicode(line[0],"utf-8")] = line[1].split(",")

def UniformPinyin(pinyin):
    pinyin, number = re.subn('(.)h(.*)', r'\1\2', pinyin)
    pinyin, number = re.subn('(.*)ng', r'\1n', pinyin)
    return pinyin

def GetPinyin(word):
    if pinyin_dict.has_key(word):
        return pinyin_dict[word][0]
    else:
        return "-"

def convertPinyin(word):
    # word = word.replace(",","")
    return map(lambda x:UniformPinyin(GetPinyin(x)),word)

def ValidCheck(wordList_py,dstWord_py):
    idx = []
    for wordidx,wordpy in enumerate(wordList_py):
        inflag = False
        for pyidx,py in enumerate(wordpy):
            if py in dstWord_py:
                dstidx = dstWord_py.index(py)
                idx.append([wordidx,pyidx,dstidx])
                inflag = True
        if not inflag:
            return []
    return idx

def PrintValidResult(pinyinSet,oriText,wordList):
    line = oriText
    try:
        dstWord_py =  convertPinyin(line)
    except:
        print "Error:%s"%(oriText)
        return
    ReplaceRule = ValidCheck(pinyinSet,dstWord_py)
    if len(ReplaceRule):
        result = list(line)
        for replace_idx in ReplaceRule:
            dst = wordList[replace_idx[0]][replace_idx[1]]
            src = result[replace_idx[2]]
            line = line.replace(src,dst)
        print line,'<<----',oriText

def ReadGushi(filename,pinyinSet,wordList):
    for (linenum,oriText) in enumerate(open(filename)):
        oriText = unicode(oriText.replace('\n',''),"utf-8")
        regexp = "<<(.*)>>:(.*)$"
        result = re.findall(regexp,oriText)
        for (title,poet) in result:
            poet = poet.replace(unicode('，',"utf-8"),'---')
            poet = poet.replace(unicode('。',"utf-8"),'---')
            poet = poet.replace(unicode('？',"utf-8"),'---')
            poet = poet.replace(unicode('、',"utf-8"),'---')
            poet = poet.replace(unicode('！',"utf-8"),'---')
            poet = poet.replace(unicode('；',"utf-8"),'---')
            poet = poet.replace(unicode('：',"utf-8"),'---')
            poet = poet.replace(unicode(':',"utf-8"),'---')
            poet = poet.replace(unicode('”',"utf-8"),'---')
            poet = poet.replace(unicode('“',"utf-8"),'---')
            poet = poet.split('---')
            for p in poet:
                PrintValidResult(pinyinSet,p,wordList)

def main(argv1,argv2):
    wordList = [argv1,argv2]
    wordList = map(lambda x:unicode(x,"gb2312"),wordList)
    pinyinSet = map(lambda x:convertPinyin(x),wordList)
    # ReadGushi("tangshi.txt",pinyinSet,wordList)
    #ReadGushi("songci.txt",pinyinSet,wordList)
    for (linenum,oriText) in enumerate(open('chengyu.txt')):
        oriText = unicode(oriText.replace('\n',''),"utf-8")
        PrintValidResult(pinyinSet,oriText,wordList)

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print "python InspirationHub.py input1 input2"
    else:
        LoadPinyinDict()
        main(sys.argv[1],sys.argv[2])<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import json, requests
from frappe import _
from frappe.model.document import Document
from six.moves.urllib.parse import urlparse

class Webhook(Document):
	def autoname(self):
		self.name = self.webhook_doctype + "-" + self.webhook_docevent
	def validate(self):
		self.validate_docevent()
		self.validate_request_url()
		self.validate_repeating_fields()
	def validate_docevent(self):
		if self.webhook_doctype:
			is_submittable = frappe.get_value("DocType", self.webhook_doctype, "is_submittable")
			if not is_submittable and self.webhook_docevent in ["on_submit", "on_cancel", "on_update_after_submit"]:
				frappe.throw(_("DocType must be Submittable for the selected Doc Event"))
	def validate_request_url(self):
		try:
			request_url = urlparse(self.request_url).netloc
			if not request_url:
				raise frappe.ValidationError
		except Exception as e:
			frappe.throw(_("Check Request URL"), exc=e)
	def validate_repeating_fields(self):
		"""Error when Same Field is entered multiple times in webhook_data"""
		webhook_data = []
		for entry in self.webhook_data:
			webhook_data.append(entry.fieldname)

		if len(webhook_data)!= len(set(webhook_data)):
			frappe.throw(_("Same Field is entered more than once"))
	def request(self, doc):
		headers = {}
		data = {}
		if self.webhook_headers:
			for h in self.webhook_headers:
				if h.get("key") and h.get("value"):
					headers[h.get("key")] = h.get("value")
		if self.webhook_data:
			for k, v in doc.as_dict().items():
				for w in self.webhook_data:
					if k == w.fieldname:
						data[w.key] = v
		r = requests.post(self.request_url, data=json.dumps(data), headers=headers, timeout=5)
		r.raise_for_status()
		frappe.logger().debug({"webhook_success":r.text})

def enqueue_webhook(doc, webhook):
	webhook = frappe.get_doc("Webhook", webhook.get("name"))
	try:
		webhook.request(doc)
	except Exception as e:
		frappe.throw(_("Error in Webhook"), exc=e)

def run_webhooks(doc, method):
	'''Run webhooks for this method'''
	if frappe.flags.in_import or frappe.flags.in_patch or frappe.flags.in_install:
		return

	if not getattr(frappe.local, 'webhooks_executed', None):
		frappe.local.webhooks_executed = []

	if doc.flags.webhooks == None:
		webhooks = frappe.cache().hget('webhooks', doc.doctype)
		if webhooks==None:
			webhooks = frappe.get_all('Webhook',
				fields=["name", "webhook_docevent"])
			frappe.cache().hset('webhooks', doc.doctype, webhooks)
		doc.flags.webhooks = webhooks

	if not doc.flags.webhooks:
		return

	def _webhook_request(webhook):
		if not webhook.name in frappe.local.webhooks_executed:
			frappe.enqueue("frappe.integrations.doctype.webhook.webhook.enqueue_webhook", doc=doc, webhook=webhook)
			frappe.local.webhooks_executed.append(webhook.name)

	event_list = ["on_update", "after_insert", "on_submit", "on_cancel", "on_trash"]

	if not doc.flags.in_insert:
		# value change is not applicable in insert
		event_list.append('on_change')
		event_list.append('before_update_after_submit')

	for webhook in doc.flags.webhooks:
		event = method if method in event_list else None
		if event and webhook.webhook_docevent == event:
			_webhook_request(webhook)
<commit_msg>Webhook retry 3 times<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import json, requests
from frappe import _
from frappe.model.document import Document
from six.moves.urllib.parse import urlparse
from time import sleep

class Webhook(Document):
	def autoname(self):
		self.name = self.webhook_doctype + "-" + self.webhook_docevent
	def validate(self):
		self.validate_docevent()
		self.validate_request_url()
		self.validate_repeating_fields()
	def validate_docevent(self):
		if self.webhook_doctype:
			is_submittable = frappe.get_value("DocType", self.webhook_doctype, "is_submittable")
			if not is_submittable and self.webhook_docevent in ["on_submit", "on_cancel", "on_update_after_submit"]:
				frappe.throw(_("DocType must be Submittable for the selected Doc Event"))
	def validate_request_url(self):
		try:
			request_url = urlparse(self.request_url).netloc
			if not request_url:
				raise frappe.ValidationError
		except Exception as e:
			frappe.throw(_("Check Request URL"), exc=e)
	def validate_repeating_fields(self):
		"""Error when Same Field is entered multiple times in webhook_data"""
		webhook_data = []
		for entry in self.webhook_data:
			webhook_data.append(entry.fieldname)

		if len(webhook_data)!= len(set(webhook_data)):
			frappe.throw(_("Same Field is entered more than once"))

def enqueue_webhook(doc, webhook):
	webhook = frappe.get_doc("Webhook", webhook.get("name"))
	headers = {}
	data = {}
	if webhook.webhook_headers:
		for h in webhook.webhook_headers:
			if h.get("key") and h.get("value"):
				headers[h.get("key")] = h.get("value")
	if webhook.webhook_data:
		for w in webhook.webhook_data:
			for k, v in doc.as_dict().items():
				if k == w.fieldname:
					data[w.key] = v
	error_status = []
	for i in range(3):
		try:
			r = requests.post(webhook.request_url, data=json.dumps(data), headers=headers, timeout=5)
			r.raise_for_status()
			frappe.logger().debug({"webhook_success":r.text})
			break
		except Exception as e:
			frappe.logger().debug({"webhook_error":e, "try": i+1})
			sleep(3*i + 1)
			if i !=2:
				continue
			else:
				raise e

def run_webhooks(doc, method):
	'''Run webhooks for this method'''
	if frappe.flags.in_import or frappe.flags.in_patch or frappe.flags.in_install:
		return

	if not getattr(frappe.local, 'webhooks_executed', None):
		frappe.local.webhooks_executed = []

	if doc.flags.webhooks == None:
		webhooks = frappe.cache().hget('webhooks', doc.doctype)
		if webhooks==None:
			webhooks = frappe.get_all('Webhook',
				fields=["name", "webhook_docevent", "webhook_doctype"])
			frappe.cache().hset('webhooks', doc.doctype, webhooks)
		doc.flags.webhooks = webhooks

	if not doc.flags.webhooks:
		return

	def _webhook_request(webhook):
		if not webhook.name in frappe.local.webhooks_executed:
			frappe.enqueue("frappe.integrations.doctype.webhook.webhook.enqueue_webhook", doc=doc, webhook=webhook)
			frappe.local.webhooks_executed.append(webhook.name)

	event_list = ["on_update", "after_insert", "on_submit", "on_cancel", "on_trash"]

	if not doc.flags.in_insert:
		# value change is not applicable in insert
		event_list.append('on_change')
		event_list.append('before_update_after_submit')

	for webhook in doc.flags.webhooks:
		event = method if method in event_list else None
		if event and webhook.webhook_docevent == event and webhook.webhook_doctype == doc.doctype:
			_webhook_request(webhook)
<|endoftext|>


<commit_before>import io
import os
import click

import frappe
from frappe.website.router import get_page_info
from frappe.website.page_renderers.base_template_page import BaseTemplatePage
from frappe.website.router import get_base_template
from frappe.website.utils import (extract_comment_tag, extract_title, get_next_link,
	get_toc, get_frontmatter, cache_html, get_sidebar_items)

WEBPAGE_PY_MODULE_PROPERTIES = ("base_template_path", "template", "no_cache", "sitemap", "condition_field")

COMMENT_PROPERTY_KEY_VALUE_MAP = {
	"no-breadcrumbs": ("no_breadcrumbs", 1),
	"show-sidebar": ("show_sidebar", 1),
	"add-breadcrumbs": ("add_breadcrumbs", 1),
	"no-header": ("no_header", 1),
	"add-next-prev-links": ("add_next_prev_links", 1),
	"no-cache": ("no_cache", 1),
	"no-sitemap": ("sitemap", 0),
	"sitemap": ("sitemap", 1)
}

class TemplatePage(BaseTemplatePage):
	def __init__(self, path, http_status_code=None):
		super().__init__(path=path, http_status_code=http_status_code)
		self.set_template_path()

	def set_template_path(self):
		'''
		Searches for file matching the path in the /www
		and /templates/pages folders and sets path if match is found
		'''
		folders = get_start_folders()
		for app in frappe.get_installed_apps(frappe_last=True):
			app_path = frappe.get_app_path(app)

			for dirname in folders:
				search_path = os.path.join(app_path, dirname, self.path)
				for file_path in self.get_index_path_options(search_path):
					if os.path.isfile(file_path):
						self.app = app
						self.app_path = app_path
						self.file_dir = dirname
						self.basename = os.path.splitext(file_path)[0]
						self.template_path = os.path.relpath(file_path, self.app_path)
						self.basepath = os.path.dirname(file_path)
						self.filename = os.path.basename(file_path)
						self.name = os.path.splitext(self.filename)[0]
						return

	def can_render(self):
		return hasattr(self, 'template_path') and bool(self.template_path)

	@staticmethod
	def get_index_path_options(search_path):
		return (frappe.as_unicode(f'{search_path}{d}') for d in ('', '.html', '.md', '/index.html', '/index.md'))

	def render(self):
		html = self.get_html()
		html = self.add_csrf_token(html)
		return self.build_response(html)

	@cache_html
	def get_html(self):
		# context object should be separate from self for security
		# because it will be accessed via the user defined template
		self.init_context()

		self.set_pymodule()
		self.update_context()
		self.setup_template()
		self.load_colocated_files()
		self.set_properties_from_source()
		self.post_process_context()

		html = self.render_template()
		html = self.update_toc(html)

		return html

	def post_process_context(self):
		self.set_user_info()
		self.add_sidebar_and_breadcrumbs()
		super(TemplatePage, self).post_process_context()

	def add_sidebar_and_breadcrumbs(self):
		if self.basepath:
			self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath)

		if self.context.add_breadcrumbs and not self.context.parents:
			parent_path = os.path.dirname(self.path)
			if self.path.endswith('index'):
				# in case of index page move one directory up for parent path
				parent_path = os.path.dirname(parent_path)

			for parent_file_path in self.get_index_path_options(parent_path):
				parent_file_path = os.path.join(self.app_path, self.file_dir, parent_file_path)
				if os.path.isfile(parent_file_path):
					parent_page_context = get_page_info(parent_file_path, self.app, self.file_dir)
					if parent_page_context:
						self.context.parents = [dict(route=os.path.dirname(self.path), title=parent_page_context.title)]
					break

	def set_pymodule(self):
		'''
		A template may have a python module with a `get_context` method along with it in the
		same folder. Also the hyphens will be coverted to underscore for python module names.
		This method sets the pymodule_name if it exists.
		'''
		template_basepath = os.path.splitext(self.template_path)[0]
		self.pymodule_name = None

		# replace - with _ in the internal modules names
		self.pymodule_path = os.path.join(os.path.dirname(template_basepath), os.path.basename(template_basepath.replace("-", "_")) + ".py")

		if os.path.exists(os.path.join(self.app_path, self.pymodule_path)):
			self.pymodule_name = self.app + "." + self.pymodule_path.replace(os.path.sep, ".")[:-3]

	def setup_template(self):
		'''Setup template source, frontmatter and markdown conversion'''
		self.source = self.get_raw_template()
		self.extract_frontmatter()
		self.convert_from_markdown()

	def update_context(self):
		self.set_page_properties()
		self.set_properties_from_source()
		self.context.build_version = frappe.utils.get_build_version()

		if self.pymodule_name:
			self.pymodule = frappe.get_module(self.pymodule_name)
			self.set_pymodule_properties()

			data = self.run_pymodule_method('get_context')
			# some methods may return a "context" object
			if data:
				self.context.update(data)
			# TODO: self.context.children = self.run_pymodule_method('get_children')

		self.context.developer_mode = frappe.conf.developer_mode
		if self.context.http_status_code:
			self.http_status_code = self.context.http_status_code

	def set_pymodule_properties(self):
		for prop in WEBPAGE_PY_MODULE_PROPERTIES:
			if hasattr(self.pymodule, prop):
				self.context[prop] = getattr(self.pymodule, prop)

	def set_page_properties(self):
		self.context.base_template = self.context.base_template \
			or get_base_template(self.path) \
			or 'templates/web.html'
		self.context.basepath = self.basepath
		self.context.basename = self.basename
		self.context.name = self.name
		self.context.path = self.path
		self.context.route = self.path
		self.context.template = self.template_path

	def set_properties_from_source(self):
		if not self.source:
			return
		context = self.context
		if not context.title:
			context.title = extract_title(self.source, self.path)

		base_template = extract_comment_tag(self.source, 'base_template')
		if base_template:
			context.base_template = base_template

		if (context.base_template
			and "{%- extends" not in self.source
			and "{% extends" not in self.source
			and "" not in self.source):
			self.source = '''{{% extends "{0}" %}}
				{{% block page_content %}}{1}{{% endblock %}}'''.format(context.base_template, self.source)

		self.set_properties_via_comments()

	def set_properties_via_comments(self):
		for comment, (context_key, value) in COMMENT_PROPERTY_KEY_VALUE_MAP.items():
			comment_tag = f""
			if comment_tag in self.source:
				self.context[context_key] = value
				click.echo(f'\n⚠  DEPRECATION WARNING: {comment_tag} will be deprecated on 2021-12-31.')
				click.echo(f'Please remove it from {self.template_path} in {self.app}')

	def run_pymodule_method(self, method_name):
		if hasattr(self.pymodule, method_name):
			try:
				import inspect
				method = getattr(self.pymodule, method_name)
				if inspect.getfullargspec(method).args:
					return method(self.context)
				else:
					return method()
			except (frappe.PermissionError, frappe.DoesNotExistError, frappe.Redirect):
				raise
			except Exception:
				if not frappe.flags.in_migrate:
					frappe.errprint(frappe.utils.get_traceback())

	def render_template(self):
		if self.source:
			html = frappe.render_template(self.source, self.context)
		elif self.template_path:
			if self.path.endswith('min.js'):
				html = self.get_raw_template() # static
			else:
				html = frappe.get_template(self.template_path).render(self.context)

		return html

	def extends_template(self):
		return (self.template_path.endswith(('.html', '.md'))
			and ('{%- extends' in self.source
				or '{% extends' in self.source))

	def get_raw_template(self):
		return frappe.get_jloader().get_source(frappe.get_jenv(), self.context.template)[0]

	def load_colocated_files(self):
		'''load co-located css/js files with the same name'''
		js_path = self.basename + '.js'
		if os.path.exists(js_path) and '{% block script %}' not in self.source:
			self.context.colocated_js = self.get_colocated_file(js_path)

		css_path = self.basename + '.css'
		if os.path.exists(css_path) and '{% block style %}' not in self.source:
			self.context.colocated_css = self.get_colocated_file(css_path)

	def get_colocated_file(self, path):
		with io.open(path, 'r', encoding = 'utf-8') as f:
			return f.read()

	def extract_frontmatter(self):
		if not self.template_path.endswith(('.md', '.html')):
			return

		try:
			# values will be used to update self
			res = get_frontmatter(self.source)
			if res['attributes']:
				self.context.update(res['attributes'])
				self.source = res['body']
		except Exception:
			pass

	def convert_from_markdown(self):
		if self.template_path.endswith('.md'):
			self.source = frappe.utils.md_to_html(self.source)
			self.context.page_toc_html = self.source.toc_html

			if not self.context.show_sidebar:
				self.source = '
' + self.source + '
'

	def update_toc(self, html):
		if '{index}' in html:
			html = html.replace('{index}', get_toc(self.path))

		if '{next}' in html:
			html = html.replace('{next}', get_next_link(self.path))

		return html

	def set_standard_path(self, path):
		self.app = 'frappe'
		self.app_path = frappe.get_app_path('frappe')
		self.path = path
		self.template_path = 'www/{path}.html'.format(path=path)

	def set_missing_values(self):
		super().set_missing_values()
		# for backward compatibility
		self.context.docs_base_url = '/docs'

	def set_user_info(self):
		from frappe.utils.user import get_fullname_and_avatar
		info = get_fullname_and_avatar(frappe.session.user)
		self.context["fullname"] = info.fullname
		self.context["user_image"] = info.avatar
		self.context["user"] = info.name

def get_start_folders():
	return frappe.local.flags.web_pages_folders or ('www', 'templates/pages')
<commit_msg>fix: Do not set base template for resource<commit_after>import io
import os
import click

import frappe
from frappe.website.router import get_page_info
from frappe.website.page_renderers.base_template_page import BaseTemplatePage
from frappe.website.router import get_base_template
from frappe.website.utils import (extract_comment_tag, extract_title, get_next_link,
	get_toc, get_frontmatter, cache_html, get_sidebar_items)

WEBPAGE_PY_MODULE_PROPERTIES = ("base_template_path", "template", "no_cache", "sitemap", "condition_field")

COMMENT_PROPERTY_KEY_VALUE_MAP = {
	"no-breadcrumbs": ("no_breadcrumbs", 1),
	"show-sidebar": ("show_sidebar", 1),
	"add-breadcrumbs": ("add_breadcrumbs", 1),
	"no-header": ("no_header", 1),
	"add-next-prev-links": ("add_next_prev_links", 1),
	"no-cache": ("no_cache", 1),
	"no-sitemap": ("sitemap", 0),
	"sitemap": ("sitemap", 1)
}

class TemplatePage(BaseTemplatePage):
	def __init__(self, path, http_status_code=None):
		super().__init__(path=path, http_status_code=http_status_code)
		self.set_template_path()

	def set_template_path(self):
		'''
		Searches for file matching the path in the /www
		and /templates/pages folders and sets path if match is found
		'''
		folders = get_start_folders()
		for app in frappe.get_installed_apps(frappe_last=True):
			app_path = frappe.get_app_path(app)

			for dirname in folders:
				search_path = os.path.join(app_path, dirname, self.path)
				for file_path in self.get_index_path_options(search_path):
					if os.path.isfile(file_path):
						self.app = app
						self.app_path = app_path
						self.file_dir = dirname
						self.basename = os.path.splitext(file_path)[0]
						self.template_path = os.path.relpath(file_path, self.app_path)
						self.basepath = os.path.dirname(file_path)
						self.filename = os.path.basename(file_path)
						self.name = os.path.splitext(self.filename)[0]
						return

	def can_render(self):
		return hasattr(self, 'template_path') and bool(self.template_path)

	@staticmethod
	def get_index_path_options(search_path):
		return (frappe.as_unicode(f'{search_path}{d}') for d in ('', '.html', '.md', '/index.html', '/index.md'))

	def render(self):
		html = self.get_html()
		html = self.add_csrf_token(html)
		return self.build_response(html)

	@cache_html
	def get_html(self):
		# context object should be separate from self for security
		# because it will be accessed via the user defined template
		self.init_context()

		self.set_pymodule()
		self.update_context()
		self.setup_template()
		self.load_colocated_files()
		self.set_properties_from_source()
		self.post_process_context()

		html = self.render_template()
		html = self.update_toc(html)

		return html

	def post_process_context(self):
		self.set_user_info()
		self.add_sidebar_and_breadcrumbs()
		super(TemplatePage, self).post_process_context()

	def add_sidebar_and_breadcrumbs(self):
		if self.basepath:
			self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath)

		if self.context.add_breadcrumbs and not self.context.parents:
			parent_path = os.path.dirname(self.path)
			if self.path.endswith('index'):
				# in case of index page move one directory up for parent path
				parent_path = os.path.dirname(parent_path)

			for parent_file_path in self.get_index_path_options(parent_path):
				parent_file_path = os.path.join(self.app_path, self.file_dir, parent_file_path)
				if os.path.isfile(parent_file_path):
					parent_page_context = get_page_info(parent_file_path, self.app, self.file_dir)
					if parent_page_context:
						self.context.parents = [dict(route=os.path.dirname(self.path), title=parent_page_context.title)]
					break

	def set_pymodule(self):
		'''
		A template may have a python module with a `get_context` method along with it in the
		same folder. Also the hyphens will be coverted to underscore for python module names.
		This method sets the pymodule_name if it exists.
		'''
		template_basepath = os.path.splitext(self.template_path)[0]
		self.pymodule_name = None

		# replace - with _ in the internal modules names
		self.pymodule_path = os.path.join(os.path.dirname(template_basepath), os.path.basename(template_basepath.replace("-", "_")) + ".py")

		if os.path.exists(os.path.join(self.app_path, self.pymodule_path)):
			self.pymodule_name = self.app + "." + self.pymodule_path.replace(os.path.sep, ".")[:-3]

	def setup_template(self):
		'''Setup template source, frontmatter and markdown conversion'''
		self.source = self.get_raw_template()
		self.extract_frontmatter()
		self.convert_from_markdown()

	def update_context(self):
		self.set_page_properties()
		self.set_properties_from_source()
		self.context.build_version = frappe.utils.get_build_version()

		if self.pymodule_name:
			self.pymodule = frappe.get_module(self.pymodule_name)
			self.set_pymodule_properties()

			data = self.run_pymodule_method('get_context')
			# some methods may return a "context" object
			if data:
				self.context.update(data)
			# TODO: self.context.children = self.run_pymodule_method('get_children')

		self.context.developer_mode = frappe.conf.developer_mode
		if self.context.http_status_code:
			self.http_status_code = self.context.http_status_code

	def set_pymodule_properties(self):
		for prop in WEBPAGE_PY_MODULE_PROPERTIES:
			if hasattr(self.pymodule, prop):
				self.context[prop] = getattr(self.pymodule, prop)

	def set_page_properties(self):
		self.context.base_template = self.context.base_template \
			or get_base_template(self.path)
		self.context.basepath = self.basepath
		self.context.basename = self.basename
		self.context.name = self.name
		self.context.path = self.path
		self.context.route = self.path
		self.context.template = self.template_path

	def set_properties_from_source(self):
		if not self.source:
			return
		context = self.context
		if not context.title:
			context.title = extract_title(self.source, self.path)

		base_template = extract_comment_tag(self.source, 'base_template')
		if base_template:
			context.base_template = base_template

		if (context.base_template
			and "{%- extends" not in self.source
			and "{% extends" not in self.source
			and "" not in self.source):
			self.source = '''{{% extends "{0}" %}}
				{{% block page_content %}}{1}{{% endblock %}}'''.format(context.base_template, self.source)

		self.set_properties_via_comments()

	def set_properties_via_comments(self):
		for comment, (context_key, value) in COMMENT_PROPERTY_KEY_VALUE_MAP.items():
			comment_tag = f""
			if comment_tag in self.source:
				self.context[context_key] = value
				click.echo(f'\n⚠  DEPRECATION WARNING: {comment_tag} will be deprecated on 2021-12-31.')
				click.echo(f'Please remove it from {self.template_path} in {self.app}')

	def run_pymodule_method(self, method_name):
		if hasattr(self.pymodule, method_name):
			try:
				import inspect
				method = getattr(self.pymodule, method_name)
				if inspect.getfullargspec(method).args:
					return method(self.context)
				else:
					return method()
			except (frappe.PermissionError, frappe.DoesNotExistError, frappe.Redirect):
				raise
			except Exception:
				if not frappe.flags.in_migrate:
					frappe.errprint(frappe.utils.get_traceback())

	def render_template(self):
		if self.source:
			html = frappe.render_template(self.source, self.context)
		elif self.template_path:
			if self.path.endswith('min.js'):
				html = self.get_raw_template() # static
			else:
				html = frappe.get_template(self.template_path).render(self.context)

		return html

	def extends_template(self):
		return (self.template_path.endswith(('.html', '.md'))
			and ('{%- extends' in self.source
				or '{% extends' in self.source))

	def get_raw_template(self):
		return frappe.get_jloader().get_source(frappe.get_jenv(), self.context.template)[0]

	def load_colocated_files(self):
		'''load co-located css/js files with the same name'''
		js_path = self.basename + '.js'
		if os.path.exists(js_path) and '{% block script %}' not in self.source:
			self.context.colocated_js = self.get_colocated_file(js_path)

		css_path = self.basename + '.css'
		if os.path.exists(css_path) and '{% block style %}' not in self.source:
			self.context.colocated_css = self.get_colocated_file(css_path)

	def get_colocated_file(self, path):
		with io.open(path, 'r', encoding = 'utf-8') as f:
			return f.read()

	def extract_frontmatter(self):
		if not self.template_path.endswith(('.md', '.html')):
			return

		try:
			# values will be used to update self
			res = get_frontmatter(self.source)
			if res['attributes']:
				self.context.update(res['attributes'])
				self.source = res['body']
		except Exception:
			pass

	def convert_from_markdown(self):
		if self.template_path.endswith('.md'):
			self.source = frappe.utils.md_to_html(self.source)
			self.context.page_toc_html = self.source.toc_html

			if not self.context.show_sidebar:
				self.source = '
' + self.source + '
'

	def update_toc(self, html):
		if '{index}' in html:
			html = html.replace('{index}', get_toc(self.path))

		if '{next}' in html:
			html = html.replace('{next}', get_next_link(self.path))

		return html

	def set_standard_path(self, path):
		self.app = 'frappe'
		self.app_path = frappe.get_app_path('frappe')
		self.path = path
		self.template_path = 'www/{path}.html'.format(path=path)

	def set_missing_values(self):
		super().set_missing_values()
		# for backward compatibility
		self.context.docs_base_url = '/docs'

	def set_user_info(self):
		from frappe.utils.user import get_fullname_and_avatar
		info = get_fullname_and_avatar(frappe.session.user)
		self.context["fullname"] = info.fullname
		self.context["user_image"] = info.avatar
		self.context["user"] = info.name

def get_start_folders():
	return frappe.local.flags.web_pages_folders or ('www', 'templates/pages')
<|endoftext|>


<commit_before>import json

from django.core.urlresolvers import reverse
from django.test import TestCase

from rest_framework import status

from bluebottle.initiatives.tests.factories import InitiativeFactory, ThemeFactory
from bluebottle.test.factory_models.accounts import BlueBottleUserFactory

class InitiativeAPITestCase(TestCase):
    """
    Integration tests for the Categories API.
    """

    def setUp(self):
        super(InitiativeAPITestCase, self).setUp()
        self.owner = BlueBottleUserFactory.create()

class InitiativeListAPITestCase(InitiativeAPITestCase):
    def setUp(self):
        super(InitiativeListAPITestCase, self).setUp()
        self.theme = ThemeFactory.create()
        self.url = reverse('initiative-list')

    def test_create(self):
        data = {
            'data': {
                'type': 'Initiative',
                'attributes': {
                    'title': 'Some title'
                },
                'relationships': {
                    'theme': {
                        'data': {
                            'type': 'Theme', 'id': self.theme.pk
                        },
                    },
                }
            }
        }
        response = self.client.post(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps(data),
            content_type="application/vnd.api+json",
            HTTP_AUTHORIZATION="JWT {0}".format(self.owner.get_jwt_token())
        )
        self.assertEqual(response.status_code, 201)
        data = json.loads(response.content)

        self.assertEqual(data['data']['attributes']['title'], 'Some title')
        self.assertEqual(
            data['data']['relationships']['owner']['data']['id'],
            unicode(self.owner.pk)
        )
        self.assertEqual(
            data['data']['relationships']['theme']['data']['id'],
            unicode(self.theme.pk)
        )

class InitiativeDetailAPITestCase(InitiativeAPITestCase):
    def setUp(self):
        super(InitiativeDetailAPITestCase, self).setUp()
        self.initiative = InitiativeFactory(
            owner=self.owner
        )

        self.url = reverse('initiative-detail', args=(self.initiative.pk, ))

    def test_put(self):
        data = {
            'data': {
                'id': self.initiative.id,
                'type': 'Initiative',
                'attributes': {
                    'title': 'Some title'
                }
            }
        }
        response = self.client.patch(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps(data),
            content_type="application/vnd.api+json",
            HTTP_AUTHORIZATION="JWT {0}".format(self.owner.get_jwt_token())
        )
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content)
        self.assertEqual(data['data']['attributes']['title'], 'Some title')

    def test_get(self):
        response = self.client.get('{}?{}'.format(self.url, 'include=owner,reviewer,theme'))
        data = json.loads(response.content)

        self.assertEqual(response.status_code, 200)

        self.assertEqual(data['data']['attributes']['title'], self.initiative.titl)
<commit_msg>Add tests<commit_after>import json

from django.core.urlresolvers import reverse
from django.test import TestCase

from rest_framework import status
from django.test import Client

from bluebottle.initiatives.models import Initiative
from bluebottle.initiatives.tests.factories import InitiativeFactory, ThemeFactory
from bluebottle.test.factory_models.accounts import BlueBottleUserFactory

class JSONAPITestClient(Client):
    def patch(self, path, data='', content_type='application/vnd.api+json', follow=False, secure=False, **extra):
        return super(JSONAPITestClient, self).put(path, data, content_type, follow, secure, **extra)

    def post(self, path, data='', content_type='application/vnd.api+json', follow=False, secure=False, **extra):
        return super(JSONAPITestClient, self).post(path, data, content_type, follow, secure, **extra)

    def generic(self, method, path, data='', content_type='application/vnd.api+json', secure=False, user=None, **extra):
        if user:
            extra['HTTP_AUTHORIZATION'] ="JWT {0}".format(user.get_jwt_token())

        return super(JSONAPITestClient, self).generic(method, path, data, content_type, secure, **extra)

class InitiativeAPITestCase(TestCase):
    """
    Integration tests for the Categories API.
    """

    def setUp(self):
        super(InitiativeAPITestCase, self).setUp()
        self.client = JSONAPITestClient()
        self.owner = BlueBottleUserFactory.create()

class InitiativeListAPITestCase(InitiativeAPITestCase):
    def setUp(self):
        super(InitiativeListAPITestCase, self).setUp()
        self.theme = ThemeFactory.create()
        self.url = reverse('initiative-list')

    def test_create(self):
        data = {
            'data': {
                'type': 'Initiative',
                'attributes': {
                    'title': 'Some title'
                },
                'relationships': {
                    'theme': {
                        'data': {
                            'type': 'Theme', 'id': self.theme.pk
                        },
                    },
                }
            }
        }
        response = self.client.post(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps(data),
            user=self.owner
        )
        self.assertEqual(response.status_code, 201)
        data = json.loads(response.content)

        initiative = Initiative.objects.get(pk=data['data']['id'])

        self.assertEqual(data['data']['attributes']['title'], initiative.title)
        self.assertEqual(
            data['data']['relationships']['owner']['data']['id'],
            unicode(self.owner.pk)
        )
        self.assertEqual(
            data['data']['relationships']['theme']['data']['id'],
            unicode(initiative.theme.pk)
        )
        self.assertEqual(len(data['included']), 2)

    def test_create_anonymous(self):
        response = self.client.post(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps({})
        )
        self.assertEqual(response.status_code, 401)

class InitiativeDetailAPITestCase(InitiativeAPITestCase):
    def setUp(self):
        super(InitiativeDetailAPITestCase, self).setUp()
        self.initiative = InitiativeFactory(
            owner=self.owner
        )

        self.url = reverse('initiative-detail', args=(self.initiative.pk, ))

    def test_patch(self):
        data = {
            'data': {
                'id': self.initiative.id,
                'type': 'Initiative',
                'attributes': {
                    'title': 'Some title'
                }
            }
        }
        response = self.client.patch(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps(data),
            user=self.owner
        )
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content)
        self.assertEqual(data['data']['attributes']['title'], 'Some title')

    def test_patch_anonymous(self):
        response = self.client.patch(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps({}),
        )
        self.assertEqual(response.status_code, 401)

    def test_patch_wrong_user(self):
        response = self.client.patch(
            '{}?{}'.format(self.url, 'include=owner,reviewer,theme'),
            json.dumps({}),
            user=BlueBottleUserFactory.create()
        )
        self.assertEqual(response.status_code, 403)

    def test_get(self):
        response = self.client.get('{}?{}'.format(self.url, 'include=owner,reviewer,theme'))
        data = json.loads(response.content)

        self.assertEqual(response.status_code, 200)

        self.assertEqual(data['data']['attributes']['title'], self.initiative.title)
        self.assertEqual(data['data']['relationships']['theme']['data']['id'], unicode(self.initiative.theme.pk))
        self.assertEqual(data['data']['relationships']['owner']['data']['id'], unicode(self.initiative.owner.pk))
        self.assertEqual(len(data['included']), 3)
<|endoftext|>