<commit_before># Copyright NuoBiT Solutions, S.L. (<https://www.nuobit.com>)
# Eric Antones <eantones@nuobit.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)

import base64
import logging

from odoo import _, fields, models
from odoo.exceptions import UserError

_logger = logging.getLogger(__name__)

class AccountInvoiceBatchProcess(models.TransientModel):
    _name = "account.invoice.batch.process"
    _description = "Account Invoice Batch Process"

    @property
    def company_id(self):
        context = self.env.context or {}
        model = context.get("active_model")
        active_ids = context.get("active_ids")

        active_objects = self.env[model].browse(active_ids)
        company = active_objects.mapped("company_id")

        if model not in ("account.invoice.batch", "account.move"):
            raise UserError(_("Unexpected model '%s'" % model))

        if len(company) > 1:
            raise UserError(
                _(
                    "More than one company found. The data selected must be of the same company"
                )
            )

        return company

    invoice_batch_sending_pdf = fields.Boolean(string="PDF", default=True)
    invoice_batch_sending_email = fields.Boolean(string="E-mail", default=True)
    invoice_batch_sending_email_template_id = fields.Many2one(
        string="E-mail template",
        comodel_name="mail.template",
        domain=[("model_id", "=", "account.move")],
        default=lambda self: self.company_id.invoice_batch_sending_email_template_id,
    )
    invoice_batch_sending_signedfacturae = fields.Boolean(
        string="Factura-e signed", default=True
    )
    invoice_batch_sending_unsignedfacturae = fields.Boolean(
        string="Factura-e unsigned", default=True
    )

    def prepare_invoices(self, invoices):
        self.ensure_one()

        # facturae
        sending_facturae = list(
            filter(
                None,
                [
                    self.invoice_batch_sending_signedfacturae
                    and "signedfacturae"
                    or None,
                    self.invoice_batch_sending_unsignedfacturae
                    and "unsignedfacturae"
                    or None,
                ],
            )
        )
        if sending_facturae:
            for inv in invoices.filtered(
                lambda x: x.invoice_batch_sending_method in sending_facturae
            ):
                if not inv.is_move_sent:
                    try:
                        invoice_file, file_name = inv.get_facturae(
                            inv.invoice_batch_sending_method == "signedfacturae"
                        )

                        file = base64.b64encode(invoice_file)
                        self.env["ir.attachment"].create(
                            {
                                "name": file_name,
                                "datas": file,
                                "res_model": "account.move",
                                "res_id": inv.id,
                                "mimetype": "application/xml",
                            }
                        )
                        inv.is_move_sent = True
                    except Exception as e:
                        _logger.debug(_("Error sending move template %s") % str(e))

        # email
        if self.invoice_batch_sending_email:
            for inv in invoices.filtered(
                lambda x: x.invoice_batch_sending_method == "email"
            ):
                if not inv.is_move_sent:
                    try:
                        inv.message_post_with_template(
                            self.invoice_batch_sending_email_template_id.id,
                            message_type="comment",
                            composition_mode="mass_mail",
                        )
                        inv.is_move_sent = True
                    except Exception as e:
                        _logger.debug(_("Error sending move template %s") % str(e))

        # pdf
        if self.invoice_batch_sending_pdf:
            return invoices.filtered(lambda x: x.invoice_batch_sending_method == "pdf")

        return self.env["account.move"]

    def process_invoices(self):
        self.ensure_one()
        context = dict(self.env.context or {})
        model = context.get("active_model")

        active_objects = self.env[model].browse(context.get("active_ids"))
        invoices_pdf = self.env["account.move"]
        if model == "account.invoice.batch":
            if not active_objects.mapped("unsent_invoice_ids"):
                raise UserError(_("There's no invoices to process"))
            for batch in active_objects:
                invoices = batch.unsent_invoice_ids
                invoices_pdf += self.prepare_invoices(invoices)
        elif model == "account.move":
            if not active_objects:
                raise UserError(_("There's no invoices to process"))
            invoices = active_objects.filtered(
                lambda x: x.invoice_batch_id and not x.is_move_sent
            )
            invoices_pdf = self.prepare_invoices(invoices)
        else:
            raise UserError(_("Unexpected model '%s'" % model))

        if invoices_pdf:
            if not self.company_id.report_service_id:
                raise UserError(_("There's no report defined on invoice company"))
            report_action = self.company_id.report_service_id.report_action(
                invoices_pdf
            )
            invoices_pdf.write({"is_move_sent": True})
            return report_action
<commit_msg>[FIX] account_invoice_batches: fix sending of facturae<commit_after># Copyright NuoBiT Solutions, S.L. (<https://www.nuobit.com>)
# Eric Antones <eantones@nuobit.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)

import base64
import logging

from odoo import _, fields, models
from odoo.exceptions import UserError

_logger = logging.getLogger(__name__)

class AccountInvoiceBatchProcess(models.TransientModel):
    _name = "account.invoice.batch.process"
    _description = "Account Invoice Batch Process"

    @property
    def company_id(self):
        context = self.env.context or {}
        model = context.get("active_model")
        active_ids = context.get("active_ids")

        active_objects = self.env[model].browse(active_ids)
        company = active_objects.mapped("company_id")

        if model not in ("account.invoice.batch", "account.move"):
            raise UserError(_("Unexpected model '%s'" % model))

        if len(company) > 1:
            raise UserError(
                _(
                    "More than one company found. The data selected must be of the same company"
                )
            )

        return company

    invoice_batch_sending_pdf = fields.Boolean(string="PDF", default=True)
    invoice_batch_sending_email = fields.Boolean(string="E-mail", default=True)
    invoice_batch_sending_email_template_id = fields.Many2one(
        string="E-mail template",
        comodel_name="mail.template",
        domain=[("model_id", "=", "account.move")],
        default=lambda self: self.company_id.invoice_batch_sending_email_template_id,
    )
    invoice_batch_sending_signedfacturae = fields.Boolean(
        string="Factura-e signed", default=True
    )
    invoice_batch_sending_unsignedfacturae = fields.Boolean(
        string="Factura-e unsigned", default=True
    )

    def prepare_invoices(self, invoices):
        self.ensure_one()

        # facturae
        sending_facturae = list(
            filter(
                None,
                [
                    self.invoice_batch_sending_signedfacturae
                    and "signedfacturae"
                    or None,
                    self.invoice_batch_sending_unsignedfacturae
                    and "unsignedfacturae"
                    or None,
                ],
            )
        )
        if sending_facturae:
            for inv in invoices.filtered(
                lambda x: x.invoice_batch_sending_method in sending_facturae
            ):
                if not inv.is_move_sent:
                    try:
                        if inv.invoice_batch_sending_method == "signedfacturae":
                            move_file = self.env.ref(
                                "l10n_es_facturae.report_facturae_signed"
                            )._render(inv.ids)[0]
                            file_name = (
                                _("facturae") + "_" + inv.name + ".xsig"
                            ).replace("/", "-")
                        else:
                            move_file = self.env.ref(
                                "l10n_es_facturae.report_facturae"
                            )._render(inv.ids)[0]
                            file_name = (
                                _("facturae") + "_" + inv.name + ".xml"
                            ).replace("/", "-")

                        file = base64.b64encode(move_file)
                        self.env["ir.attachment"].create(
                            {
                                "name": file_name,
                                "datas": file,
                                "res_model": "account.move",
                                "res_id": inv.id,
                                "mimetype": "application/xml",
                            }
                        )
                        inv.is_move_sent = True
                    except Exception as e:
                        _logger.debug(_("Error sending move template %s") % str(e))

        # email
        if self.invoice_batch_sending_email:
            for inv in invoices.filtered(
                lambda x: x.invoice_batch_sending_method == "email"
            ):
                if not inv.is_move_sent:
                    try:
                        inv.message_post_with_template(
                            self.invoice_batch_sending_email_template_id.id,
                            message_type="comment",
                            composition_mode="mass_mail",
                        )
                        inv.is_move_sent = True
                    except Exception as e:
                        _logger.debug(_("Error sending move template %s") % str(e))

        # pdf
        if self.invoice_batch_sending_pdf:
            return invoices.filtered(lambda x: x.invoice_batch_sending_method == "pdf")

        return self.env["account.move"]

    def process_invoices(self):
        self.ensure_one()
        context = dict(self.env.context or {})
        model = context.get("active_model")

        active_objects = self.env[model].browse(context.get("active_ids"))
        invoices_pdf = self.env["account.move"]
        if model == "account.invoice.batch":
            if not active_objects.mapped("unsent_invoice_ids"):
                raise UserError(_("There's no invoices to process"))
            for batch in active_objects:
                invoices = batch.unsent_invoice_ids
                invoices_pdf += self.prepare_invoices(invoices)
        elif model == "account.move":
            if not active_objects:
                raise UserError(_("There's no invoices to process"))
            invoices = active_objects.filtered(
                lambda x: x.invoice_batch_id and not x.is_move_sent
            )
            invoices_pdf = self.prepare_invoices(invoices)
        else:
            raise UserError(_("Unexpected model '%s'" % model))

        if invoices_pdf:
            if not self.company_id.report_service_id:
                raise UserError(_("There's no report defined on invoice company"))
            report_action = self.company_id.report_service_id.report_action(
                invoices_pdf
            )
            invoices_pdf.write({"is_move_sent": True})
            return report_action
<|endoftext|>


<commit_before># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Binary creation support functions."""

load(
    "@build_bazel_rules_apple//apple/bundling:entitlements.bzl",
    "entitlements",
)
load(
    "@build_bazel_rules_apple//apple/bundling:product_support.bzl",
    "product_support",
)
load(
    "@build_bazel_rules_apple//common:providers.bzl",
    "providers",
)
load(
    "@build_bazel_rules_apple//apple/bundling:platform_support.bzl",
    "platform_support",
)

def _get_binary_provider(deps, provider_key):
  """Returns the provider from a rule's binary dependency.

  Bundling rules depend on binary rules via the "deps" attribute, which
  canonically supports a label list. This function validates that the
  "deps" attribute has only a single value, as is expected for bundling
  rules, before extracting and returning the provider of the given key.

  Args:
    deps: The list of the target's dependencies.
    provider_key: The key of the provider to return.

  Returns:
    The provider propagated by the single "deps" target of the current rule.
  """
  matching_providers = providers.find_all(deps, provider_key)
  if matching_providers:
    if len(matching_providers) > 1:
      fail("Expected only one binary provider")
    return matching_providers[0]
  return None

def _create_stub_binary(ctx):
  """Creates a binary for a bundle by copying a stub from the SDK.

  Some Apple bundles may not need a binary target depending on their product
  type; for example, watchOS applications and iMessage sticker packs contain
  stub binaries copied from the platform SDK, rather than binaries with user
  code. This function creates an `apple_stub_binary` target (instead of
  `apple_binary`) that ensures that the platform transition is correct (for
  platform selection in downstream dependencies) but that does not cause any
  user code to be linked.

  Args:
    ctx: The bundling rule context.

  Returns:
    The binary stub artifact copied out of the SDK.
  """
  product_type = product_support.product_type(ctx)
  product_type_descriptor = product_support.product_type_descriptor(
      product_type)

  if not product_type_descriptor or not product_type_descriptor.stub:
    fail(('%s failed: create_stub_binary may only be called from a bundling ' +
         'rule of a product type which declares a stub') % ctx.label)

  output_artifact = ctx.actions.declare_file(
      "stubs/%s.binary_stub" % (ctx.label.name))

  platform_support.xcode_env_action(
      ctx,
      outputs=[output_artifact],
      executable=ctx.executable._xcrunwrapper,
      arguments=["/bin/cp",
                 product_type_descriptor.stub.xcenv_based_path,
                 output_artifact.path],
      mnemonic="CopyStubExecutable",
      progress_message="Copying stub executable for %s" % (ctx.label)
  )

  return output_artifact

def _create_stub_binary_target(
    name,
    platform_type,
    stub_descriptor,
    **kwargs):
  """Creates a binary target for a bundle by copying a stub from the SDK.

  Some Apple bundles may not need a binary target depending on their product
  type; for example, watchOS applications and iMessage sticker packs contain
  stub binaries copied from the platform SDK, rather than binaries with user
  code. This function creates an `apple_stub_binary` target (instead of
  `apple_binary`) that ensures that the platform transition is correct (for
  platform selection in downstream dependencies) but that does not cause any
  user code to be linked.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be copied.
    stub_descriptor: The information about the product type's stub executable.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.

  Deprecated:
    Stub binary targets are deprecated. Use create_stub_binary in the bundling
    rule context instead.
  """
  # TODO(b/69630422): Migrate all callers off of apple_stub_binary. Use
  # create_stub_binary in the rule context instead.

  bundling_args = dict(kwargs)

  apple_binary_name = "%s.apple_binary" % name
  minimum_os_version = kwargs.get("minimum_os_version")

  # Remove the deps so that we only pass them to the binary, not to the
  # bundling rule.
  deps = bundling_args.pop("deps", [])

  native.apple_stub_binary(
      name = apple_binary_name,
      minimum_os_version = minimum_os_version,
      platform_type = platform_type,
      xcenv_based_path = stub_descriptor.xcenv_based_path,
      deps = deps,
      tags = ["manual"] + kwargs.get("tags", []),
      testonly = kwargs.get("testonly"),
      visibility = kwargs.get("visibility"),
  )

  bundling_args["binary"] = apple_binary_name
  bundling_args["deps"] = [apple_binary_name]

  # For device builds, make sure that the stub binary still gets signed with the
  # appropriate entitlements (and that they have their substitutions applied).
  entitlements_value = kwargs.get("entitlements")
  provisioning_profile = kwargs.get("provisioning_profile")
  entitlements_name = "%s_entitlements" % name
  entitlements(
      name = entitlements_name,
      bundle_id = kwargs.get("bundle_id"),
      entitlements = entitlements_value,
      platform_type = platform_type,
      provisioning_profile = provisioning_profile,
      validation_mode = kwargs.get("entitlements_validation"),
  )
  bundling_args["entitlements"] = ":" + entitlements_name

  return bundling_args

def _entitlement_args_for_stub(
    name,
    platform_type,
    **kwargs):
  """Creates an entitlements target for a bundle needing only an SDK stub.

  Apple bundles not needing a binary target, for example, watchOS applications
  and iMessage sticker packs, only contain stub binaries copied from the
  platform SDK rather than binaries with user code. This function creates
  only an entitlements target to ensure the stub binary gets appropriately
  signed.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be copied.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  bundling_args = dict(kwargs)

  entitlements_value = kwargs.get("entitlements")
  provisioning_profile = kwargs.get("provisioning_profile")
  entitlements_name = "%s_entitlements" % name
  entitlements(
      name = entitlements_name,
      bundle_id = kwargs.get("bundle_id"),
      entitlements = entitlements_value,
      platform_type = platform_type,
      provisioning_profile = provisioning_profile,
      validation_mode = kwargs.get("entitlements_validation"),
  )
  bundling_args["entitlements"] = ":" + entitlements_name
  # This is required by the configuration transition on the 'deps' attribute
  # of the bundling rule.
  bundling_args["platform_type"] = platform_type

  return bundling_args

def _create_linked_binary_target(
    name,
    platform_type,
    linkopts,
    binary_type="executable",
    sdk_frameworks=[],
    extension_safe=False,
    bundle_loader=None,
    suppress_entitlements=False,
    **kwargs):
  """Creates a binary target for a bundle by linking user code.

  This function also wraps the entitlements handling logic. It returns a
  modified copy of the given keyword arguments that has `binary` and
  `entitlements` attributes added if necessary and removes other
  binary-specific options (such as `linkopts`).

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be built.
    linkopts: Extra linking options to be passed to the binary target.
    binary_type: The type of binary to create. Can be "executable",
        "loadable_bundle" or "dylib".
    sdk_frameworks: Additional SDK frameworks that should be linked with the
        final binary.
    extension_safe: If true, compiles and links this framework with
        '-application-extension', restricting the binary to use only
        extension-safe APIs. False by default.
    bundle_loader: Label to an apple_binary target that will act as the
        bundle_loader for this apple_binary. Can only be set if binary_type is
        "loadable_bundle".
    suppress_entitlements: True/False, indicates that the entitlements() should
        be suppressed.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  bundling_args = dict(kwargs)

  minimum_os_version = kwargs.get("minimum_os_version")
  provisioning_profile = kwargs.get("provisioning_profile")

  if suppress_entitlements:
    entitlements_deps = []
  else:
    entitlements_value = bundling_args.pop("entitlements", None)
    entitlements_name = "%s_entitlements" % name
    entitlements(
        name = entitlements_name,
        bundle_id = kwargs.get("bundle_id"),
        entitlements = entitlements_value,
        platform_type = platform_type,
        provisioning_profile = provisioning_profile,
        validation_mode = kwargs.get("entitlements_validation"),
    )
    bundling_args["entitlements"] = ":" + entitlements_name
    entitlements_deps = [":" + entitlements_name]

  # Remove the deps so that we only pass them to the binary, not to the
  # bundling rule.
  deps = bundling_args.pop("deps", [])

  # Link the executable from any library deps provided. Pass the entitlements
  # target as an extra dependency to the binary rule to pick up the extra
  # linkopts (if any) propagated by it.
  apple_binary_name = "%s.apple_binary" % name
  native.apple_binary(
      name = apple_binary_name,
      binary_type = binary_type,
      bundle_loader = bundle_loader,
      dylibs = kwargs.get("frameworks"),
      extension_safe = extension_safe,
      features = kwargs.get("features"),
      linkopts = linkopts + ["-rpath", "@executable_path/../../Frameworks"],
      minimum_os_version = minimum_os_version,
      platform_type = platform_type,
      sdk_frameworks = sdk_frameworks,
      deps = deps,
      # TODO(b/64611007): We use non-propagated deps as a workaround for now to
      # ensure that the linkopts and link_inputs for the entitlements don't get
      # propagated to dependent binaries, like test bundles.
      non_propagated_deps = entitlements_deps,
      tags = ["manual"] + kwargs.get("tags", []),
      testonly = kwargs.get("testonly"),
      visibility = kwargs.get("visibility"),
  )
  bundling_args["binary"] = apple_binary_name
  bundling_args["deps"] = [":" + apple_binary_name]

  return bundling_args

def _create_binary(name, platform_type, suppress_entitlements=False, **kwargs):
  """Creates a binary target for a bundle.

  This function checks the desired product type of the bundle and creates either
  an `apple_binary` or `apple_stub_binary` depending on what the product type
  needs. It must be called from one of the top-level application or extension
  macros, because it invokes a rule to create a target. As such, it cannot be
  called within rule implementation functions.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be built.
    suppress_entitlements: True/False, indicates that the entitlements() should
        be suppressed.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  args_copy = dict(kwargs)

  binary_type = args_copy.pop("binary_type", "executable")
  linkopts = args_copy.pop("linkopts", [])
  sdk_frameworks = args_copy.pop("sdk_frameworks", [])
  extension_safe = args_copy.pop("extension_safe", False)
  bundle_loader = args_copy.pop("bundle_loader", None)

  # If a user provides a "binary" attribute of their own, it is ignored and
  # silently overwritten below. Instead of allowing this, we should fail fast
  # to prevent confusion.
  if "binary" in args_copy:
    fail("Do not provide your own binary; one will be linked from your deps.",
         attr="binary")

  # Note the pop/get difference here. If the attribute is present as "private",
  # we want to pop it off so that it does not get passed down to the underlying
  # bundling rule (this is the macro's way of giving us default information in
  # the rule that we don't have access to yet). If the argument is present
  # without the underscore, then we leave it in so that the bundling rule can
  # access the value the user provided in their build target (if any).
  product_type = args_copy.pop("_product_type", None)
  if not product_type:
    product_type = args_copy.get("product_type")

  product_type_descriptor = product_support.product_type_descriptor(
      product_type)
  if product_type_descriptor and product_type_descriptor.stub:
    if suppress_entitlements:
      fail("INTERNAL ERROR: suppress_entitlements unsupported on stubs.")
    return _create_stub_binary_target(
        name, platform_type, product_type_descriptor.stub, **args_copy)
  else:
    return _create_linked_binary_target(
        name, platform_type, linkopts, binary_type, sdk_frameworks,
        extension_safe, bundle_loader, suppress_entitlements=suppress_entitlements,
        **args_copy)

# Define the loadable module that lists the exported symbols in this file.
binary_support = struct(
    create_binary=_create_binary,
    create_stub_binary=_create_stub_binary,
    entitlement_args_for_stub=_entitlement_args_for_stub,
    get_binary_provider=_get_binary_provider,
)
<commit_msg>Remove unnecessary usage of non_propagated_deps.<commit_after># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Binary creation support functions."""

load(
    "@build_bazel_rules_apple//apple/bundling:entitlements.bzl",
    "entitlements",
)
load(
    "@build_bazel_rules_apple//apple/bundling:product_support.bzl",
    "product_support",
)
load(
    "@build_bazel_rules_apple//common:providers.bzl",
    "providers",
)
load(
    "@build_bazel_rules_apple//apple/bundling:platform_support.bzl",
    "platform_support",
)

def _get_binary_provider(deps, provider_key):
  """Returns the provider from a rule's binary dependency.

  Bundling rules depend on binary rules via the "deps" attribute, which
  canonically supports a label list. This function validates that the
  "deps" attribute has only a single value, as is expected for bundling
  rules, before extracting and returning the provider of the given key.

  Args:
    deps: The list of the target's dependencies.
    provider_key: The key of the provider to return.

  Returns:
    The provider propagated by the single "deps" target of the current rule.
  """
  matching_providers = providers.find_all(deps, provider_key)
  if matching_providers:
    if len(matching_providers) > 1:
      fail("Expected only one binary provider")
    return matching_providers[0]
  return None

def _create_stub_binary(ctx):
  """Creates a binary for a bundle by copying a stub from the SDK.

  Some Apple bundles may not need a binary target depending on their product
  type; for example, watchOS applications and iMessage sticker packs contain
  stub binaries copied from the platform SDK, rather than binaries with user
  code. This function creates an `apple_stub_binary` target (instead of
  `apple_binary`) that ensures that the platform transition is correct (for
  platform selection in downstream dependencies) but that does not cause any
  user code to be linked.

  Args:
    ctx: The bundling rule context.

  Returns:
    The binary stub artifact copied out of the SDK.
  """
  product_type = product_support.product_type(ctx)
  product_type_descriptor = product_support.product_type_descriptor(
      product_type)

  if not product_type_descriptor or not product_type_descriptor.stub:
    fail(('%s failed: create_stub_binary may only be called from a bundling ' +
         'rule of a product type which declares a stub') % ctx.label)

  output_artifact = ctx.actions.declare_file(
      "stubs/%s.binary_stub" % (ctx.label.name))

  platform_support.xcode_env_action(
      ctx,
      outputs=[output_artifact],
      executable=ctx.executable._xcrunwrapper,
      arguments=["/bin/cp",
                 product_type_descriptor.stub.xcenv_based_path,
                 output_artifact.path],
      mnemonic="CopyStubExecutable",
      progress_message="Copying stub executable for %s" % (ctx.label)
  )

  return output_artifact

def _create_stub_binary_target(
    name,
    platform_type,
    stub_descriptor,
    **kwargs):
  """Creates a binary target for a bundle by copying a stub from the SDK.

  Some Apple bundles may not need a binary target depending on their product
  type; for example, watchOS applications and iMessage sticker packs contain
  stub binaries copied from the platform SDK, rather than binaries with user
  code. This function creates an `apple_stub_binary` target (instead of
  `apple_binary`) that ensures that the platform transition is correct (for
  platform selection in downstream dependencies) but that does not cause any
  user code to be linked.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be copied.
    stub_descriptor: The information about the product type's stub executable.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.

  Deprecated:
    Stub binary targets are deprecated. Use create_stub_binary in the bundling
    rule context instead.
  """
  # TODO(b/69630422): Migrate all callers off of apple_stub_binary. Use
  # create_stub_binary in the rule context instead.

  bundling_args = dict(kwargs)

  apple_binary_name = "%s.apple_binary" % name
  minimum_os_version = kwargs.get("minimum_os_version")

  # Remove the deps so that we only pass them to the binary, not to the
  # bundling rule.
  deps = bundling_args.pop("deps", [])

  native.apple_stub_binary(
      name = apple_binary_name,
      minimum_os_version = minimum_os_version,
      platform_type = platform_type,
      xcenv_based_path = stub_descriptor.xcenv_based_path,
      deps = deps,
      tags = ["manual"] + kwargs.get("tags", []),
      testonly = kwargs.get("testonly"),
      visibility = kwargs.get("visibility"),
  )

  bundling_args["binary"] = apple_binary_name
  bundling_args["deps"] = [apple_binary_name]

  # For device builds, make sure that the stub binary still gets signed with the
  # appropriate entitlements (and that they have their substitutions applied).
  entitlements_value = kwargs.get("entitlements")
  provisioning_profile = kwargs.get("provisioning_profile")
  entitlements_name = "%s_entitlements" % name
  entitlements(
      name = entitlements_name,
      bundle_id = kwargs.get("bundle_id"),
      entitlements = entitlements_value,
      platform_type = platform_type,
      provisioning_profile = provisioning_profile,
      validation_mode = kwargs.get("entitlements_validation"),
  )
  bundling_args["entitlements"] = ":" + entitlements_name

  return bundling_args

def _entitlement_args_for_stub(
    name,
    platform_type,
    **kwargs):
  """Creates an entitlements target for a bundle needing only an SDK stub.

  Apple bundles not needing a binary target, for example, watchOS applications
  and iMessage sticker packs, only contain stub binaries copied from the
  platform SDK rather than binaries with user code. This function creates
  only an entitlements target to ensure the stub binary gets appropriately
  signed.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be copied.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  bundling_args = dict(kwargs)

  entitlements_value = kwargs.get("entitlements")
  provisioning_profile = kwargs.get("provisioning_profile")
  entitlements_name = "%s_entitlements" % name
  entitlements(
      name = entitlements_name,
      bundle_id = kwargs.get("bundle_id"),
      entitlements = entitlements_value,
      platform_type = platform_type,
      provisioning_profile = provisioning_profile,
      validation_mode = kwargs.get("entitlements_validation"),
  )
  bundling_args["entitlements"] = ":" + entitlements_name
  # This is required by the configuration transition on the 'deps' attribute
  # of the bundling rule.
  bundling_args["platform_type"] = platform_type

  return bundling_args

def _create_linked_binary_target(
    name,
    platform_type,
    linkopts,
    binary_type="executable",
    sdk_frameworks=[],
    extension_safe=False,
    bundle_loader=None,
    suppress_entitlements=False,
    **kwargs):
  """Creates a binary target for a bundle by linking user code.

  This function also wraps the entitlements handling logic. It returns a
  modified copy of the given keyword arguments that has `binary` and
  `entitlements` attributes added if necessary and removes other
  binary-specific options (such as `linkopts`).

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be built.
    linkopts: Extra linking options to be passed to the binary target.
    binary_type: The type of binary to create. Can be "executable",
        "loadable_bundle" or "dylib".
    sdk_frameworks: Additional SDK frameworks that should be linked with the
        final binary.
    extension_safe: If true, compiles and links this framework with
        '-application-extension', restricting the binary to use only
        extension-safe APIs. False by default.
    bundle_loader: Label to an apple_binary target that will act as the
        bundle_loader for this apple_binary. Can only be set if binary_type is
        "loadable_bundle".
    suppress_entitlements: True/False, indicates that the entitlements() should
        be suppressed.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  bundling_args = dict(kwargs)

  minimum_os_version = kwargs.get("minimum_os_version")
  provisioning_profile = kwargs.get("provisioning_profile")

  if suppress_entitlements:
    entitlements_deps = []
  else:
    entitlements_value = bundling_args.pop("entitlements", None)
    entitlements_name = "%s_entitlements" % name
    entitlements(
        name = entitlements_name,
        bundle_id = kwargs.get("bundle_id"),
        entitlements = entitlements_value,
        platform_type = platform_type,
        provisioning_profile = provisioning_profile,
        validation_mode = kwargs.get("entitlements_validation"),
    )
    bundling_args["entitlements"] = ":" + entitlements_name
    entitlements_deps = [":" + entitlements_name]

  # Remove the deps so that we only pass them to the binary, not to the
  # bundling rule.
  deps = bundling_args.pop("deps", [])

  # Link the executable from any library deps provided. Pass the entitlements
  # target as an extra dependency to the binary rule to pick up the extra
  # linkopts (if any) propagated by it.
  apple_binary_name = "%s.apple_binary" % name
  native.apple_binary(
      name = apple_binary_name,
      binary_type = binary_type,
      bundle_loader = bundle_loader,
      dylibs = kwargs.get("frameworks"),
      extension_safe = extension_safe,
      features = kwargs.get("features"),
      linkopts = linkopts + ["-rpath", "@executable_path/../../Frameworks"],
      minimum_os_version = minimum_os_version,
      platform_type = platform_type,
      sdk_frameworks = sdk_frameworks,
      deps = deps + entitlements_deps,
      tags = ["manual"] + kwargs.get("tags", []),
      testonly = kwargs.get("testonly"),
      visibility = kwargs.get("visibility"),
  )
  bundling_args["binary"] = apple_binary_name
  bundling_args["deps"] = [":" + apple_binary_name]

  return bundling_args

def _create_binary(name, platform_type, suppress_entitlements=False, **kwargs):
  """Creates a binary target for a bundle.

  This function checks the desired product type of the bundle and creates either
  an `apple_binary` or `apple_stub_binary` depending on what the product type
  needs. It must be called from one of the top-level application or extension
  macros, because it invokes a rule to create a target. As such, it cannot be
  called within rule implementation functions.

  Args:
    name: The name of the bundle target, from which the binary target's name
        will be derived.
    platform_type: The platform type for which the binary should be built.
    suppress_entitlements: True/False, indicates that the entitlements() should
        be suppressed.
    **kwargs: The arguments that were passed into the top-level macro.

  Returns:
    A modified copy of `**kwargs` that should be passed to the bundling rule.
  """
  args_copy = dict(kwargs)

  binary_type = args_copy.pop("binary_type", "executable")
  linkopts = args_copy.pop("linkopts", [])
  sdk_frameworks = args_copy.pop("sdk_frameworks", [])
  extension_safe = args_copy.pop("extension_safe", False)
  bundle_loader = args_copy.pop("bundle_loader", None)

  # If a user provides a "binary" attribute of their own, it is ignored and
  # silently overwritten below. Instead of allowing this, we should fail fast
  # to prevent confusion.
  if "binary" in args_copy:
    fail("Do not provide your own binary; one will be linked from your deps.",
         attr="binary")

  # Note the pop/get difference here. If the attribute is present as "private",
  # we want to pop it off so that it does not get passed down to the underlying
  # bundling rule (this is the macro's way of giving us default information in
  # the rule that we don't have access to yet). If the argument is present
  # without the underscore, then we leave it in so that the bundling rule can
  # access the value the user provided in their build target (if any).
  product_type = args_copy.pop("_product_type", None)
  if not product_type:
    product_type = args_copy.get("product_type")

  product_type_descriptor = product_support.product_type_descriptor(
      product_type)
  if product_type_descriptor and product_type_descriptor.stub:
    if suppress_entitlements:
      fail("INTERNAL ERROR: suppress_entitlements unsupported on stubs.")
    return _create_stub_binary_target(
        name, platform_type, product_type_descriptor.stub, **args_copy)
  else:
    return _create_linked_binary_target(
        name, platform_type, linkopts, binary_type, sdk_frameworks,
        extension_safe, bundle_loader, suppress_entitlements=suppress_entitlements,
        **args_copy)

# Define the loadable module that lists the exported symbols in this file.
binary_support = struct(
    create_binary=_create_binary,
    create_stub_binary=_create_stub_binary,
    entitlement_args_for_stub=_entitlement_args_for_stub,
    get_binary_provider=_get_binary_provider,
)
<|endoftext|>


<commit_before>'''
Importing this package will set the security context appropriately for
use with the :func:`dxpy.DXHTTPRequest` function, which will then set
the headers appropriately when communicating with the API server.  In
addition, it will set the default workspace according to
DX_WORKSPACE_ID (if running inside an Execution Environment) or
DX_PROJECT_CONTEXT_ID (otherwise).  This workspace will be used by
default for any object handler methods that require a project ID.

When importing the :mod:`dxpy.bindings` submodule, the methods for
setting the security context etc. do not need to be used directly so
long as the appropriate environment variables DX_APISERVER_HOST,
DX_APISERVER_PORT, and DX_SECURITY_CONTEXT have been properly set.

If an HTTP/HTTPS proxy is to be used, set the environment variables
beforehand as applicable while using the format 'hostname:port'
(e.g. '10.10.1.10:3128'):

* **HTTP_PROXY**: 'hostname:port' for the HTTP proxy
* **HTTPS_PROXY**: 'hostname:port' for the HTTPS proxy

'''

import os, json, requests
from requests.exceptions import ConnectionError, HTTPError
from requests.auth import AuthBase
from dxpy.exceptions import *

API_VERSION = '1.0.0'
AUTH_HELPER = None
JOB_ID, WORKSPACE_ID, PROJECT_CONTEXT_ID = None, None, None

MAX_RETRIES = 0

def DXHTTPRequest(resource, data, method='POST', headers={}, auth=None, config=None, jsonify_data=True, want_full_response=False, **kwargs):
    '''
    :param resource: API server route, e.g. "/record/new"
    :type resource: string
    :param data: Contents for the request body
    :param jsonify_data: Indicates whether *data* should be converted from a Python list or dict to a JSON string
    :type jsonify_data: boolean
    :param want_full_response: Indicates whether the function should return the full :class:`requests.Response` object or just the content of the response
    :type want_full_response: boolean
    :returns: Response from API server in the requested format.  Note: if *want_full_response* is set to False and the header "content-type" is found in the response with value "application/json", the contents of the response will **always** be converted from JSON to Python before it is returned, and it will therefore be of type list or dict.
    :raises: :exc:`requests.exceptions.HTTPError` if response code was not 200 (OK), :exc:`ValueError` if the response from the API server cannot be decoded

    Wrapper around requests.request(). Inserts authentication and
    converts *data* to JSON.
    '''
    url = APISERVER + resource

    if auth is None and AUTH_HELPER is None:
        # TODO: support for unauthenticated requests to unprivileged routes
        raise DXError("Unable to send request without authentication")
    if auth is None:
        auth = AUTH_HELPER
    if config is None:
        config = {}
    # TODO: decide which routes are safe to retry
    # TODO: exponential backoff policy in requests
    config.setdefault('max_retries', MAX_RETRIES)
    if 'Content-Type' not in headers:
        headers['Content-Type'] = 'application/json'
    if jsonify_data:
        data = json.dumps(data)

    headers['DNAnexus-API'] = API_VERSION

    last_error = None
    for retry in range(MAX_RETRIES + 1):
        try:
            response = requests.request(method, url, data=data, headers=headers,
                                        auth=auth, config=config, **kwargs)

            # If HTTP code that is not 200 (OK) is received and the content is
            # JSON, parse it and throw the appropriate error.  Otherwise,
            # raise the usual exception.
            if response.status_code != requests.codes.ok:
                # response.headers key lookup is case-insensitive
                if response.headers.get('content-type', '').startswith('application/json'):
                    content = json.loads(response.content)
                    raise DXAPIError(content["error"]["type"],
                                     content["error"]["message"],
                                     response.status_code)
                response.raise_for_status()

            if want_full_response:
                return response
            else:
                if 'content-length' in response.headers:
                    if int(response.headers['content-length']) != len(response.content):
                        raise HTTPError("Received response with content-length header set to %s but content length is %d"
                            % (response.headers['content-length'], len(response.content)))

                if response.headers.get('content-type', '').startswith('application/json'):
                    return json.loads(response.content)
                return response.content
        except ConnectionError as e:
            last_error = e
        except HTTPError as e:
            last_error = e
            if method != 'GET' and response.status_code != requests.codes.server_error:
                break # Disable retries
        if last_error is None:
            last_error = DXError("Internal error in DXHTTPRequest")
    raise last_error

class DXHTTPOAuth2(AuthBase):
    def __init__(self, security_context):
        self.security_context = security_context

    def __call__(self, r):
        if self.security_context["auth_token_type"].lower() == 'bearer':
            r.headers['Authorization'] = \
                self.security_context["auth_token_type"] + " " + \
                self.security_context["auth_token"]
        else:
            raise NotImplementedError("Token types other than bearer are not yet supported")
        return r

def set_api_server_info(host='localhost', port=8124, protocol='http'):
    global APISERVER_HOST, APISERVER_PORT, APISERVER
    APISERVER_HOST = host
    APISERVER_PORT = port
    APISERVER = protocol + "://" + host + ":" + str(port)

def set_security_context(security_context):
    global SECURITY_CONTEXT, AUTH_HELPER
    SECURITY_CONTEXT = security_context
    AUTH_HELPER = DXHTTPOAuth2(security_context)

def set_job_id(dxid):
    """
    :param id: ID of a job
    :type id: string

    Sets the ID of the running job.  TODO: Not clear yet that this is
    necessary.

    """
    global JOB_ID
    JOB_ID = dxid

def set_workspace_id(dxid):
    """
    :param id: ID of a project or workspace
    :type id: string

    Sets the default project or workspace for object creation and
    modification to *id*.
    """

    global WORKSPACE_ID
    WORKSPACE_ID = dxid

def set_project_context(dxid):
    """
    :param id: Project ID
    :type id: string

    Sets the project context for a running job.

    """

    global PROJECT_CONTEXT_ID
    PROJECT_CONTEXT_ID = dxid

if "DX_APISERVER_HOST" in os.environ and "DX_APISERVER_PORT" in os.environ:
    set_api_server_info(host=os.environ["DX_APISERVER_HOST"], port=os.environ["DX_APISERVER_PORT"])
else:
    set_api_server_info()

if "DX_SECURITY_CONTEXT" in os.environ:
    set_security_context(json.loads(os.environ['DX_SECURITY_CONTEXT']))
else:
    print "Warning: no security context found in environment variables"

if "DX_JOB_ID" in os.environ:
    set_job_id(os.environ["DX_JOB_ID"])
    if "DX_WORKSPACE_ID" in os.environ:
        set_workspace_id(os.environ["DX_WORKSPACE_ID"])
    if "DX_PROJECT_CONTEXT_ID" in os.environ:
        set_project_context(os.environ["DX_PROJECT_CONTEXT_ID"])
else:
    if "DX_PROJECT_CONTEXT_ID" in os.environ:
        set_workspace_id(os.environ["DX_PROJECT_CONTEXT_ID"])

from dxpy.bindings import *
from dxpy.dxlog import *
<commit_msg>actually retry on server_error<commit_after>'''
Importing this package will set the security context appropriately for
use with the :func:`dxpy.DXHTTPRequest` function, which will then set
the headers appropriately when communicating with the API server.  In
addition, it will set the default workspace according to
DX_WORKSPACE_ID (if running inside an Execution Environment) or
DX_PROJECT_CONTEXT_ID (otherwise).  This workspace will be used by
default for any object handler methods that require a project ID.

When importing the :mod:`dxpy.bindings` submodule, the methods for
setting the security context etc. do not need to be used directly so
long as the appropriate environment variables DX_APISERVER_HOST,
DX_APISERVER_PORT, and DX_SECURITY_CONTEXT have been properly set.

If an HTTP/HTTPS proxy is to be used, set the environment variables
beforehand as applicable while using the format 'hostname:port'
(e.g. '10.10.1.10:3128'):

* **HTTP_PROXY**: 'hostname:port' for the HTTP proxy
* **HTTPS_PROXY**: 'hostname:port' for the HTTPS proxy

'''

import os, json, requests
from requests.exceptions import ConnectionError, HTTPError
from requests.auth import AuthBase
from dxpy.exceptions import *

API_VERSION = '1.0.0'
AUTH_HELPER = None
JOB_ID, WORKSPACE_ID, PROJECT_CONTEXT_ID = None, None, None

MAX_RETRIES = 0

def DXHTTPRequest(resource, data, method='POST', headers={}, auth=None, config=None, jsonify_data=True, want_full_response=False, **kwargs):
    '''
    :param resource: API server route, e.g. "/record/new"
    :type resource: string
    :param data: Contents for the request body
    :param jsonify_data: Indicates whether *data* should be converted from a Python list or dict to a JSON string
    :type jsonify_data: boolean
    :param want_full_response: Indicates whether the function should return the full :class:`requests.Response` object or just the content of the response
    :type want_full_response: boolean
    :returns: Response from API server in the requested format.  Note: if *want_full_response* is set to False and the header "content-type" is found in the response with value "application/json", the contents of the response will **always** be converted from JSON to Python before it is returned, and it will therefore be of type list or dict.
    :raises: :exc:`requests.exceptions.HTTPError` if response code was not 200 (OK), :exc:`ValueError` if the response from the API server cannot be decoded

    Wrapper around requests.request(). Inserts authentication and
    converts *data* to JSON.
    '''
    url = APISERVER + resource

    if auth is None and AUTH_HELPER is None:
        # TODO: support for unauthenticated requests to unprivileged routes
        raise DXError("Unable to send request without authentication")
    if auth is None:
        auth = AUTH_HELPER
    if config is None:
        config = {}
    # TODO: decide which routes are safe to retry
    # TODO: exponential backoff policy in requests
    config.setdefault('max_retries', MAX_RETRIES)
    if 'Content-Type' not in headers:
        headers['Content-Type'] = 'application/json'
    if jsonify_data:
        data = json.dumps(data)

    headers['DNAnexus-API'] = API_VERSION

    last_error = None
    for retry in range(MAX_RETRIES + 1):
        try:
            response = requests.request(method, url, data=data, headers=headers,
                                        auth=auth, config=config, **kwargs)

            # If HTTP code that is not 200 (OK) is received and the content is
            # JSON, parse it and throw the appropriate error.  Otherwise,
            # raise the usual exception.
            if response.status_code != requests.codes.ok:
                # response.headers key lookup is case-insensitive
                if response.headers.get('content-type', '').startswith('application/json'):
                    content = json.loads(response.content)
                    raise DXAPIError(content["error"]["type"],
                                     content["error"]["message"],
                                     response.status_code)
                response.raise_for_status()

            if want_full_response:
                return response
            else:
                if 'content-length' in response.headers:
                    if int(response.headers['content-length']) != len(response.content):
                        raise HTTPError("Received response with content-length header set to %s but content length is %d"
                            % (response.headers['content-length'], len(response.content)))

                if response.headers.get('content-type', '').startswith('application/json'):
                    return json.loads(response.content)
                return response.content
        except ConnectionError as e:
            last_error = e
        except (DXAPIError, HTTPError) as e:
            last_error = e
            if method != 'GET' and response.status_code != requests.codes.server_error:
                break # Disable retries
        if last_error is None:
            last_error = DXError("Internal error in DXHTTPRequest")
    raise last_error

class DXHTTPOAuth2(AuthBase):
    def __init__(self, security_context):
        self.security_context = security_context

    def __call__(self, r):
        if self.security_context["auth_token_type"].lower() == 'bearer':
            r.headers['Authorization'] = \
                self.security_context["auth_token_type"] + " " + \
                self.security_context["auth_token"]
        else:
            raise NotImplementedError("Token types other than bearer are not yet supported")
        return r

def set_api_server_info(host='localhost', port=8124, protocol='http'):
    global APISERVER_HOST, APISERVER_PORT, APISERVER
    APISERVER_HOST = host
    APISERVER_PORT = port
    APISERVER = protocol + "://" + host + ":" + str(port)

def set_security_context(security_context):
    global SECURITY_CONTEXT, AUTH_HELPER
    SECURITY_CONTEXT = security_context
    AUTH_HELPER = DXHTTPOAuth2(security_context)

def set_job_id(dxid):
    """
    :param id: ID of a job
    :type id: string

    Sets the ID of the running job.  TODO: Not clear yet that this is
    necessary.

    """
    global JOB_ID
    JOB_ID = dxid

def set_workspace_id(dxid):
    """
    :param id: ID of a project or workspace
    :type id: string

    Sets the default project or workspace for object creation and
    modification to *id*.
    """

    global WORKSPACE_ID
    WORKSPACE_ID = dxid

def set_project_context(dxid):
    """
    :param id: Project ID
    :type id: string

    Sets the project context for a running job.

    """

    global PROJECT_CONTEXT_ID
    PROJECT_CONTEXT_ID = dxid

if "DX_APISERVER_HOST" in os.environ and "DX_APISERVER_PORT" in os.environ:
    set_api_server_info(host=os.environ["DX_APISERVER_HOST"], port=os.environ["DX_APISERVER_PORT"])
else:
    set_api_server_info()

if "DX_SECURITY_CONTEXT" in os.environ:
    set_security_context(json.loads(os.environ['DX_SECURITY_CONTEXT']))
else:
    print "Warning: no security context found in environment variables"

if "DX_JOB_ID" in os.environ:
    set_job_id(os.environ["DX_JOB_ID"])
    if "DX_WORKSPACE_ID" in os.environ:
        set_workspace_id(os.environ["DX_WORKSPACE_ID"])
    if "DX_PROJECT_CONTEXT_ID" in os.environ:
        set_project_context(os.environ["DX_PROJECT_CONTEXT_ID"])
else:
    if "DX_PROJECT_CONTEXT_ID" in os.environ:
        set_workspace_id(os.environ["DX_PROJECT_CONTEXT_ID"])

from dxpy.bindings import *
from dxpy.dxlog import *
<|endoftext|>


<commit_before>"""Tasks we perform on the master server.

See `askmaster.py` for tasks that are run on minions."""

import os, time
import aws
from fabric.api import sudo, local
from buildercore import core, bootstrap, config, keypair, project, utils
from buildercore.utils import last, lmap, exsubdict
from decorators import debugtask, echo_output, requires_project, requires_aws_stack, requires_feature
from kids.cache import cache as cached
import logging

LOG = logging.getLogger(__name__)

#
#
#

@debugtask
@requires_feature('write-keypairs-to-s3')
def write_missing_keypairs_to_s3():
    "uploads any missing ec2 keys to S3 if they're present locally"
    remote_keys = keypair.all_in_s3()
    local_paths = keypair.all_locally()
    local_keys = lmap(os.path.basename, local_paths)

    to_upload = set(local_keys).difference(set(remote_keys))

    print('remote:', remote_keys)
    print('local:', local_keys)
    print('to upload:', to_upload)

    def write(key):
        stackname = os.path.splitext(key)[0]
        keypair.write_keypair_to_s3(stackname)

    lmap(write, to_upload)

def write_missing_context_to_s3():
    pass

@debugtask
@requires_feature('write-keypairs-to-s3')
@requires_aws_stack
@echo_output
def download_keypair(stackname):
    try:
        path = keypair.download_from_s3(stackname)
        local('chmod 400 -R %s' % path)
        return path
    except EnvironmentError as err:
        LOG.info(err)

#
#
#

@debugtask
@echo_output
@cached
def server_access():
    """returns True if this builder instance has access to the master server.
    access may be available through presence of the master-server's bootstrap user's
    identify file OR current user is in master server's allowed_keys list"""
    stackname = core.find_master(core.find_region())
    public_ip = core.stack_data(stackname, ensure_single_instance=True)[0]['ip_address']
    result = local('ssh -o "StrictHostKeyChecking no" %s@%s "exit"' % (config.BOOTSTRAP_USER, public_ip))
    return result.return_code == 0

@echo_output
def aws_update_many_projects(pname_list):
    minions = ' or '.join([pname + "-*" for pname in pname_list])
    region = aws.find_region()
    with core.stack_conn(core.find_master(region)):
        sudo("salt -C '%s' state.highstate --retcode-passthrough" % minions)

@debugtask
@requires_project
def aws_update_projects(pname):
    "calls state.highstate on ALL projects matching <projectname>-*"
    return aws_update_many_projects([pname])

@debugtask
@requires_aws_stack
def remaster_minion(stackname, new_master_stackname):
    "tell minion who their new master is. deletes any existing master key on minion"
    # TODO: turn this into a decorator
    import cfn
    instances = cfn._check_want_to_be_running(stackname, 1)

    master_ip = core.stack_data(new_master_stackname)[0]['private_ip_address']

    print('re-mastering %s to %s' % (stackname, master_ip))

    def work():
        sudo("rm -f /etc/salt/pki/minion/minion_master.pub")  # destroy the old master key we have
        sudo("sed -i -e 's/^master:.*$/master: %s/g' /etc/salt/minion" % master_ip)
    core.stack_all_ec2_nodes(stackname, work, username=config.BOOTSTRAP_USER)
    bootstrap.update_ec2_stack(stackname, concurrency='serial', master_ip=master_ip)

@debugtask
@requires_aws_stack
def remaster_all_minions(new_master_stackname):
    import cfn
    LOG.info('new master is: %s', new_master_stackname)
    ec2stacks = project.ec2_projects()
    ignore = [
        'master-server',
        'jats4r',
    ]
    ec2stacks = exsubdict(ec2stacks, ignore)
    pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically

    # only update ec2 instances in the same region as the new master
    region = aws.find_region(new_master_stackname)
    active_stacks = core.active_stack_names(region)
    stack_idx = utils.mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)

    def sortbyenv(n):
        adhoc = 0 # do these first
        order = {
            'continuumtest': 1,
            'ci': 2,
            'end2end': 3,
            'prod': 4, # update prod last
        }
        return order.get(core.parse_stackname(n)[-1], adhoc)

    remastered_list = open('remastered.txt', 'r').readlines() if os.path.exists('remastered.txt') else []
    for pname in pname_list:
        if pname not in stack_idx:
            continue
        stack_list = sorted(stack_idx[pname], key=sortbyenv)
        LOG.info("%r instances: %s" % (pname, ", ".join(stack_list)))
        try:
            for stackname in stack_list:
                try:
                    if stackname in remastered_list:
                        LOG.info("already updated: %s", stackname)
                        continue
                    LOG.info("*" * 80)
                    LOG.info("updating: %s" % stackname)
                    cfn.update_template(stackname)
                    remaster_minion(stackname, new_master_stackname)
                    open('remastered.txt', 'a').write("%s\n" % stackname)
                except KeyboardInterrupt:
                    LOG.warn("skipping stack: %s", stackname)
                    time.sleep(1)
                except:
                    LOG.exception("unhandled exception updating stack: %s", stackname)
        except KeyboardInterrupt:
            LOG.warn("quitting")
            break

    LOG.info("wrote 'remastered.txt'")
<commit_msg>updated labels in bulk remaster<commit_after>"""Tasks we perform on the master server.

See `askmaster.py` for tasks that are run on minions."""

import os, time
import aws
from fabric.api import sudo, local
from buildercore import core, bootstrap, config, keypair, project, utils
from buildercore.utils import last, lmap, exsubdict
from decorators import debugtask, echo_output, requires_project, requires_aws_stack, requires_feature
from kids.cache import cache as cached
import logging

LOG = logging.getLogger(__name__)

#
#
#

@debugtask
@requires_feature('write-keypairs-to-s3')
def write_missing_keypairs_to_s3():
    "uploads any missing ec2 keys to S3 if they're present locally"
    remote_keys = keypair.all_in_s3()
    local_paths = keypair.all_locally()
    local_keys = lmap(os.path.basename, local_paths)

    to_upload = set(local_keys).difference(set(remote_keys))

    print('remote:', remote_keys)
    print('local:', local_keys)
    print('to upload:', to_upload)

    def write(key):
        stackname = os.path.splitext(key)[0]
        keypair.write_keypair_to_s3(stackname)

    lmap(write, to_upload)

def write_missing_context_to_s3():
    pass

@debugtask
@requires_feature('write-keypairs-to-s3')
@requires_aws_stack
@echo_output
def download_keypair(stackname):
    try:
        path = keypair.download_from_s3(stackname)
        local('chmod 400 -R %s' % path)
        return path
    except EnvironmentError as err:
        LOG.info(err)

#
#
#

@debugtask
@echo_output
@cached
def server_access():
    """returns True if this builder instance has access to the master server.
    access may be available through presence of the master-server's bootstrap user's
    identify file OR current user is in master server's allowed_keys list"""
    stackname = core.find_master(core.find_region())
    public_ip = core.stack_data(stackname, ensure_single_instance=True)[0]['ip_address']
    result = local('ssh -o "StrictHostKeyChecking no" %s@%s "exit"' % (config.BOOTSTRAP_USER, public_ip))
    return result.return_code == 0

@echo_output
def aws_update_many_projects(pname_list):
    minions = ' or '.join([pname + "-*" for pname in pname_list])
    region = aws.find_region()
    with core.stack_conn(core.find_master(region)):
        sudo("salt -C '%s' state.highstate --retcode-passthrough" % minions)

@debugtask
@requires_project
def aws_update_projects(pname):
    "calls state.highstate on ALL projects matching <projectname>-*"
    return aws_update_many_projects([pname])

@debugtask
@requires_aws_stack
def remaster_minion(stackname, new_master_stackname):
    "tell minion who their new master is. deletes any existing master key on minion"
    # TODO: turn this into a decorator
    import cfn
    instances = cfn._check_want_to_be_running(stackname, 1)

    master_ip = core.stack_data(new_master_stackname)[0]['private_ip_address']

    print('re-mastering %s to %s' % (stackname, master_ip))

    def work():
        sudo("rm -f /etc/salt/pki/minion/minion_master.pub")  # destroy the old master key we have
        sudo("sed -i -e 's/^master:.*$/master: %s/g' /etc/salt/minion" % master_ip)
    core.stack_all_ec2_nodes(stackname, work, username=config.BOOTSTRAP_USER)
    bootstrap.update_ec2_stack(stackname, concurrency='serial', master_ip=master_ip)

@debugtask
@requires_aws_stack
def remaster_all_minions(new_master_stackname):
    import cfn
    LOG.info('new master is: %s', new_master_stackname)
    ec2stacks = project.ec2_projects()
    ignore = [
        'master-server',
        'jats4r',
    ]
    ec2stacks = exsubdict(ec2stacks, ignore)
    pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically

    # only update ec2 instances in the same region as the new master
    region = aws.find_region(new_master_stackname)
    active_stacks = core.active_stack_names(region)
    stack_idx = utils.mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)

    def sortbyenv(n):
        adhoc = 0 # do these first
        order = {
            'continuumtest': 1,
            'ci': 2,
            'end2end': 3,
            'prod': 4, # update prod last
        }
        return order.get(core.parse_stackname(n)[-1], adhoc)

    remastered_list = open('remastered.txt', 'r').readlines() if os.path.exists('remastered.txt') else []
    for pname in pname_list:
        if pname not in stack_idx:
            continue
        stack_list = sorted(stack_idx[pname], key=sortbyenv)
        LOG.info("%r instances: %s" % (pname, ", ".join(stack_list)))
        try:
            for stackname in stack_list:
                try:
                    if stackname in remastered_list:
                        LOG.info("already updated, skipping stack: %s", stackname)
                        continue
                    LOG.info("*" * 80)
                    LOG.info("updating: %s" % stackname)
                    cfn.update_template(stackname)
                    remaster_minion(stackname, new_master_stackname)
                    open('remastered.txt', 'a').write("%s\n" % stackname)
                except KeyboardInterrupt:
                    LOG.warn("ctrl-c, skipping stack: %s", stackname)
                    time.sleep(1)
                except:
                    LOG.exception("unhandled exception updating stack: %s", stackname)
        except KeyboardInterrupt:
            LOG.warn("quitting")
            break

    LOG.info("wrote 'remastered.txt'")
<|endoftext|>


<commit_before>
# import glob
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
# import os
# import pandas as pd
# import pylab as pl
# import re
# import sys
# import trappy

# Configure logging
import logging

class Filters(object):

    def __init__(self, trace, tasks=None):
        self.trace = trace
        self.tasks = tasks

        self.big_tasks = {}

        self.big_frequent_tasks_pids = None
        self.wkp_frequent_tasks_pids = None

        self.big_cap = self.trace.platform['nrg_model']['big']['cpu']['cap_max']
        self.little_cap = self.trace.platform['nrg_model']['little']['cpu']['cap_max']
        self.big_cpus = self.trace.platform['clusters']['big']
        self.little_cpus = self.trace.platform['clusters']['little']

        # Minimum and Maximum x_time to use for all plots
        self.x_min = 0
        self.x_max = self.trace.time_range

        # Reset x axis time range to full scale
        self.setXTimeRange()

    def setXTimeRange(self, t_min=None, t_max=None):
        if t_min is None:
            self.x_min = 0
        else:
            self.x_min = t_min
        if t_max is None:
            self.x_max = self.trace.time_range
        else:
            self.x_max = t_max
        logging.info('Set plots time range to (%.6f, %.6f)[s]',
                self.x_min, self.x_max)

    def topBigTasks(self, max_tasks=10, min_samples=100, min_utilization=None):
        """
        Tasks which had a 'utilization' bigger than the specified threshold
        """

        if min_utilization is None:
            min_utilization = self.little_cap

        df = self.trace.df('tload')
        big_tasks_events = df[df.util_avg > min_utilization]
        big_tasks = big_tasks_events.pid.unique()

        big_tasks_count = big_tasks.size
        print 'Total {} tasks with at least {} "utilization" samples > {}'\
                .format(big_tasks_count, min_samples, min_utilization)

        big_tasks_stats = big_tasks_events.groupby('pid')\
                            .describe(include=['object']);
        big_tasks_pids = big_tasks_stats.unstack()['comm']\
                            .sort(columns=['count'], ascending=False)
        big_tasks_pids = big_tasks_pids[big_tasks_pids['count'] > min_samples]

        big_topmost = big_tasks_pids.head(max_tasks)
        print 'Top {} "big" tasks:'.format(max_tasks)
        print big_topmost

        self.big_frequent_tasks_pids = list(big_topmost.index)

        # Keep track of big tasks tload events
        self.big_tasks['tload'] = big_tasks_events
        return self.big_frequent_tasks_pids

    def _taskIsBig(self, utilization):
        if utilization > self.little_cap:
            return self.big_cap
        return self.little_cap

    def plotBigTasks(self, max_tasks=10, min_samples=100, min_utilization=None):

        # Get the list of big and frequent tasks
        if self.big_frequent_tasks_pids is None:
            pids = self.topBigTasks(max_tasks, min_samples, min_utilization)

        big_frequent_tasks_count = len(self.big_frequent_tasks_pids)
        if big_frequent_tasks_count == 0:
            print "No big/frequent tasks to plot"
            return

        # Get the list of events for all big frequent tasks
        df = self.trace.df('tload')
        big_frequent_tasks_events = df[df.pid.isin(self.big_frequent_tasks_pids)]

        # Add a column to represent big status
        big_frequent_tasks_events.loc[:,'isbig'] = \
                big_frequent_tasks_events['util_avg'].map(self._taskIsBig)

        # Define axes for side-by-side plottings
        fig, axes = plt.subplots(big_frequent_tasks_count, 1,
                                 figsize=(14, big_frequent_tasks_count*5));
        plt.subplots_adjust(wspace=0.1, hspace=0.2);

        plot_idx = 0
        for i, group in big_frequent_tasks_events.groupby('pid'):

            # Build task names (there could be multiple, during the task lifetime)
            big_frequent_task_i = big_frequent_tasks_events[big_frequent_tasks_events['pid'] == i]
            task_names = big_frequent_task_i.comm.unique()
            task_name = 'PID: ' + str(i)
            for s in task_names:
                task_name += ' | ' + s

            # Plot title
            if (big_frequent_tasks_count == 1):
                ax_ratio = axes
            else:
                 ax_ratio = axes[plot_idx]
            ax_ratio.set_title(task_name);

            # Left axis: utilization
            ax_ratio = group.plot(y=['util_avg', 'isbig'],
                            style=['r.', '-b'],
                            drawstyle='steps-post',
                            linewidth=1,
                            ax=ax_ratio)
            ax_ratio.set_xlim(self.x_min, self.x_max);
            ax_ratio.set_ylim(0, 1100)
            ax_ratio.set_ylabel('util_avg')

            plot_idx+=1

        print 'Tasks which have been a "utilization" of {0:d} for at least {1:d} samples'\
            .format(self.little_cap, min_samples)

    def topWakeupTasks(self, max_tasks=10, min_wakeups=100):
        """
        Tasks which wakeups more frequent than a specified threshold
        """

        df = self.trace.df('swkp')

        wkp_tasks_stats = df.groupby('pid').describe(include=['object'])
        wkp_tasks_pids = wkp_tasks_stats.unstack()['comm']\
                        .sort(columns=['count'], ascending=False)
        wkp_tasks_pids = wkp_tasks_pids[wkp_tasks_pids['count'] > min_wakeups]

        wkp_topmost = wkp_tasks_pids.head(max_tasks)
        print 'Top {} "big" tasks:'.format(max_tasks)
        print wkp_topmost

        self.wkp_frequent_tasks_pids = list(wkp_topmost.index)
        return self.wkp_frequent_tasks_pids

    def plotWakeupTasks(self, max_tasks=10, min_wakeups=0, per_cluster=False):

        # Get the list of big and frequent tasks
        if self.wkp_frequent_tasks_pids is None:
            pids = self.topWakeupTasks(max_tasks, min_wakeups)

        wkp_frequent_tasks_count = len(self.wkp_frequent_tasks_pids)
        if wkp_frequent_tasks_count == 0:
            print "No big/frequent wakeups tasks to plot"
            return

        # Define axes for side-by-side plottings
        fig, axes = plt.subplots(2, 1, figsize=(14, 5));
        plt.subplots_adjust(wspace=0.2, hspace=0.3);

        if per_cluster:

            # Get per cluster wakeup events
            df = self.trace.df('swkpn')
            big_frequent = (
                    (df.target_cpu.isin(self.big_cpus))
                    )
            ntbc = df[big_frequent]
            little_frequent = (
                    (df.target_cpu.isin(self.little_cpus))
                    )
            ntlc = df[little_frequent];

            ax = axes[0]
            ax.set_title('Tasks Forks on big CPUs');
            ax.set_xlim(self.x_min, self.x_max);
            ntbc.pid.plot(style=['g.'], ax=ax);

            ax = axes[1]
            ax.set_title('Tasks Forks on LITTLE CPUs');
            ax.set_xlim(self.x_min, self.x_max);
            ntlc.pid.plot(style=['g.'], ax=ax);

        else:

            ax = axes[0]
            ax.set_title('Tasks WakeUps Events');
            df = self.trace.df('swkp')
            df.pid.astype(int).plot(style=['b.'], ax=ax);
            ax.set_xlim(self.x_min, self.x_max);
            ax.xaxis.set_visible(False);

            ax = axes[1]
            ax.set_title('Tasks Forks Events');
            df = self.trace.df('swkpn')
            df.pid.astype(int).plot(style=['r.'], ax=ax);
            ax.set_xlim(self.x_min, self.x_max);
            ax.xaxis.set_visible(False);

    def plotTasksVsFrequency(self, big_cluster=True):

        if big_cluster:
            cluster_correct = 'big'
            cluster_wrong = 'LITTLE'
            cpus = self.big_cpus
        else:
            cluster_correct = 'LITTLE'
            cluster_wrong = 'big'
            cpus = self.little_cpus

        fig, axes = plt.subplots(2, 1, figsize=(14, 5));
        plt.subplots_adjust(wspace=0.2, hspace=0.3);

        df_wkp  = self.big_tasks['tload']
        # Add column of expected cluster, depending on utilization value and
        # capacity of the selected cluster
        bu_bc = ( \
                (df_wkp['util_avg'] > 500) & \
                (df_wkp['cpu'].isin([2,3]))
            )
        su_lc = ( \
                (df_wkp['util_avg'] <= 500) & \
                (df_wkp['cpu'].isin([0,1]))
            )
        df_wkp.loc[:,'ccap_mutil'] = np.select(
            [(bu_bc | su_lc)], [True], False)

        df_freq = self.trace.df('pfreq')
        rd_freq = df_freq[df_freq['cpu'].isin(cpus)]

        ax = axes[0]
        ax.set_title('Big Tasks Utilization vs Allocation');
        for ucolor, umatch in zip('gr', [True, False]):
            cdata  = df_wkp[df_wkp['ccap_mutil'] == umatch]
            if (len(cdata) > 0):
                cdata['util_avg'].plot(ax=ax,
                        style=[ucolor+'.'], legend=False);
        ax.set_xlim(self.x_min, self.x_max);
        ax.xaxis.set_visible(False);

        ax = axes[1]
        ax.set_title('Frequencies on "{}" cluster'.format(cluster_correct))
        df_freq['frequency'].plot(style=['-b'], ax=ax, drawstyle='steps-post');
        ax.set_xlim(self.x_min, self.x_max);

    def rtTasks(self, max_prio = 100):
        df = self.trace.df('sswitch')
        df = df[df.next_prio <= max_prio]
        df = df[['next_pid', 'next_comm']]
        df = df.drop_duplicates()
        rt_tasks = {}
        for pid,task in df.values:
                rt_tasks[pid] = task
        return rt_tasks
<commit_msg>libs/utils/filters: fix pandas warnings on deprecated methods usage<commit_after>
# import glob
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
# import os
# import pandas as pd
# import pylab as pl
# import re
# import sys
# import trappy

# Configure logging
import logging

class Filters(object):

    def __init__(self, trace, tasks=None):
        self.trace = trace
        self.tasks = tasks

        self.big_tasks = {}

        self.big_frequent_tasks_pids = None
        self.wkp_frequent_tasks_pids = None

        self.big_cap = self.trace.platform['nrg_model']['big']['cpu']['cap_max']
        self.little_cap = self.trace.platform['nrg_model']['little']['cpu']['cap_max']
        self.big_cpus = self.trace.platform['clusters']['big']
        self.little_cpus = self.trace.platform['clusters']['little']

        # Minimum and Maximum x_time to use for all plots
        self.x_min = 0
        self.x_max = self.trace.time_range

        # Reset x axis time range to full scale
        self.setXTimeRange()

    def setXTimeRange(self, t_min=None, t_max=None):
        if t_min is None:
            self.x_min = 0
        else:
            self.x_min = t_min
        if t_max is None:
            self.x_max = self.trace.time_range
        else:
            self.x_max = t_max
        logging.info('Set plots time range to (%.6f, %.6f)[s]',
                self.x_min, self.x_max)

    def topBigTasks(self, max_tasks=10, min_samples=100, min_utilization=None):
        """
        Tasks which had a 'utilization' bigger than the specified threshold
        """

        if min_utilization is None:
            min_utilization = self.little_cap

        df = self.trace.df('tload')
        big_tasks_events = df[df.util_avg > min_utilization]
        big_tasks = big_tasks_events.pid.unique()

        big_tasks_count = big_tasks.size
        print 'Total {} tasks with at least {} "utilization" samples > {}'\
                .format(big_tasks_count, min_samples, min_utilization)

        big_tasks_stats = big_tasks_events.groupby('pid')\
                            .describe(include=['object']);
        big_tasks_pids = big_tasks_stats.unstack()['comm']\
                            .sort_values(by=['count'], ascending=False)
        big_tasks_pids = big_tasks_pids[big_tasks_pids['count'] > min_samples]

        big_topmost = big_tasks_pids.head(max_tasks)
        print 'Top {} "big" tasks:'.format(max_tasks)
        print big_topmost

        self.big_frequent_tasks_pids = list(big_topmost.index)

        # Keep track of big tasks tload events
        self.big_tasks['tload'] = big_tasks_events
        return self.big_frequent_tasks_pids

    def _taskIsBig(self, utilization):
        if utilization > self.little_cap:
            return self.big_cap
        return self.little_cap

    def plotBigTasks(self, max_tasks=10, min_samples=100, min_utilization=None):

        # Get the list of big and frequent tasks
        if self.big_frequent_tasks_pids is None:
            pids = self.topBigTasks(max_tasks, min_samples, min_utilization)

        big_frequent_tasks_count = len(self.big_frequent_tasks_pids)
        if big_frequent_tasks_count == 0:
            print "No big/frequent tasks to plot"
            return

        # Get the list of events for all big frequent tasks
        df = self.trace.df('tload')
        big_frequent_tasks_events = df[df.pid.isin(self.big_frequent_tasks_pids)]

        # Add a column to represent big status
        big_frequent_tasks_events.loc[:,'isbig'] = \
                big_frequent_tasks_events['util_avg'].map(self._taskIsBig)

        # Define axes for side-by-side plottings
        fig, axes = plt.subplots(big_frequent_tasks_count, 1,
                                 figsize=(14, big_frequent_tasks_count*5));
        plt.subplots_adjust(wspace=0.1, hspace=0.2);

        plot_idx = 0
        for i, group in big_frequent_tasks_events.groupby('pid'):

            # Build task names (there could be multiple, during the task lifetime)
            big_frequent_task_i = big_frequent_tasks_events[big_frequent_tasks_events['pid'] == i]
            task_names = big_frequent_task_i.comm.unique()
            task_name = 'PID: ' + str(i)
            for s in task_names:
                task_name += ' | ' + s

            # Plot title
            if (big_frequent_tasks_count == 1):
                ax_ratio = axes
            else:
                 ax_ratio = axes[plot_idx]
            ax_ratio.set_title(task_name);

            # Left axis: utilization
            ax_ratio = group.plot(y=['util_avg', 'isbig'],
                            style=['r.', '-b'],
                            drawstyle='steps-post',
                            linewidth=1,
                            ax=ax_ratio)
            ax_ratio.set_xlim(self.x_min, self.x_max);
            ax_ratio.set_ylim(0, 1100)
            ax_ratio.set_ylabel('util_avg')

            plot_idx+=1

        print 'Tasks which have been a "utilization" of {0:d} for at least {1:d} samples'\
            .format(self.little_cap, min_samples)

    def topWakeupTasks(self, max_tasks=10, min_wakeups=100):
        """
        Tasks which wakeups more frequent than a specified threshold
        """

        df = self.trace.df('swkp')

        wkp_tasks_stats = df.groupby('pid').describe(include=['object'])
        wkp_tasks_pids = wkp_tasks_stats.unstack()['comm']\
                        .sort_values(by=['count'], ascending=False)
        wkp_tasks_pids = wkp_tasks_pids[wkp_tasks_pids['count'] > min_wakeups]

        wkp_topmost = wkp_tasks_pids.head(max_tasks)
        print 'Top {} "big" tasks:'.format(max_tasks)
        print wkp_topmost

        self.wkp_frequent_tasks_pids = list(wkp_topmost.index)
        return self.wkp_frequent_tasks_pids

    def plotWakeupTasks(self, max_tasks=10, min_wakeups=0, per_cluster=False):

        # Get the list of big and frequent tasks
        if self.wkp_frequent_tasks_pids is None:
            pids = self.topWakeupTasks(max_tasks, min_wakeups)

        wkp_frequent_tasks_count = len(self.wkp_frequent_tasks_pids)
        if wkp_frequent_tasks_count == 0:
            print "No big/frequent wakeups tasks to plot"
            return

        # Define axes for side-by-side plottings
        fig, axes = plt.subplots(2, 1, figsize=(14, 5));
        plt.subplots_adjust(wspace=0.2, hspace=0.3);

        if per_cluster:

            # Get per cluster wakeup events
            df = self.trace.df('swkpn')
            big_frequent = (
                    (df.target_cpu.isin(self.big_cpus))
                    )
            ntbc = df[big_frequent]
            little_frequent = (
                    (df.target_cpu.isin(self.little_cpus))
                    )
            ntlc = df[little_frequent];

            ax = axes[0]
            ax.set_title('Tasks Forks on big CPUs');
            ax.set_xlim(self.x_min, self.x_max);
            ntbc.pid.plot(style=['g.'], ax=ax);

            ax = axes[1]
            ax.set_title('Tasks Forks on LITTLE CPUs');
            ax.set_xlim(self.x_min, self.x_max);
            ntlc.pid.plot(style=['g.'], ax=ax);

        else:

            ax = axes[0]
            ax.set_title('Tasks WakeUps Events');
            df = self.trace.df('swkp')
            df.pid.astype(int).plot(style=['b.'], ax=ax);
            ax.set_xlim(self.x_min, self.x_max);
            ax.xaxis.set_visible(False);

            ax = axes[1]
            ax.set_title('Tasks Forks Events');
            df = self.trace.df('swkpn')
            df.pid.astype(int).plot(style=['r.'], ax=ax);
            ax.set_xlim(self.x_min, self.x_max);
            ax.xaxis.set_visible(False);

    def plotTasksVsFrequency(self, big_cluster=True):

        if big_cluster:
            cluster_correct = 'big'
            cluster_wrong = 'LITTLE'
            cpus = self.big_cpus
        else:
            cluster_correct = 'LITTLE'
            cluster_wrong = 'big'
            cpus = self.little_cpus

        fig, axes = plt.subplots(2, 1, figsize=(14, 5));
        plt.subplots_adjust(wspace=0.2, hspace=0.3);

        df_wkp  = self.big_tasks['tload']
        # Add column of expected cluster, depending on utilization value and
        # capacity of the selected cluster
        bu_bc = ( \
                (df_wkp['util_avg'] > 500) & \
                (df_wkp['cpu'].isin([2,3]))
            )
        su_lc = ( \
                (df_wkp['util_avg'] <= 500) & \
                (df_wkp['cpu'].isin([0,1]))
            )
        df_wkp.loc[:,'ccap_mutil'] = np.select(
            [(bu_bc | su_lc)], [True], False)

        df_freq = self.trace.df('pfreq')
        rd_freq = df_freq[df_freq['cpu'].isin(cpus)]

        ax = axes[0]
        ax.set_title('Big Tasks Utilization vs Allocation');
        for ucolor, umatch in zip('gr', [True, False]):
            cdata  = df_wkp[df_wkp['ccap_mutil'] == umatch]
            if (len(cdata) > 0):
                cdata['util_avg'].plot(ax=ax,
                        style=[ucolor+'.'], legend=False);
        ax.set_xlim(self.x_min, self.x_max);
        ax.xaxis.set_visible(False);

        ax = axes[1]
        ax.set_title('Frequencies on "{}" cluster'.format(cluster_correct))
        df_freq['frequency'].plot(style=['-b'], ax=ax, drawstyle='steps-post');
        ax.set_xlim(self.x_min, self.x_max);

    def rtTasks(self, max_prio = 100):
        df = self.trace.df('sswitch')
        df = df[df.next_prio <= max_prio]
        df = df[['next_pid', 'next_comm']]
        df = df.drop_duplicates()
        rt_tasks = {}
        for pid,task in df.values:
                rt_tasks[pid] = task
        return rt_tasks
<|endoftext|>


<commit_before>from decimal import Decimal
from .types import Color, Distance

def darken(color, amount):
    if not isinstance(color, Color):
        raise ValueError('Cannot darken non-color:', str(color))

    if isinstance(amount, Decimal):
        new_r = int(color._r - amount)
        new_g = int(color._g - amount)
        new_b = int(color._b - amount)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Color):
        new_r = int(color._r - amount._r)
        new_g = int(color._g - amount._g)
        new_b = int(color._b - amount._b)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Distance) and amount.unit == '%':
        new_r = int(color._r * (100 - amount.value) / 100)
        new_g = int(color._r * (100 - amount.value) / 100)
        new_b = int(color._r * (100 - amount.value) / 100)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    raise ValueError('Cannot darken by value:', str(amount))

def lighten(color, amount):
    if not isinstance(color, Color):
        raise ValueError('Cannot lighten non-color:', str(color))

    if isinstance(amount, Decimal):
        new_r = int(color._r + amount)
        new_g = int(color._g + amount)
        new_b = int(color._b + amount)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Color):
        new_r = int(color._r + amount._r)
        new_g = int(color._g + amount._g)
        new_b = int(color._b + amount._b)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Distance) and amount.unit == '%':
        new_r = int(color._r * (100 + amount.value) / 100)
        new_g = int(color._r * (100 + amount.value) / 100)
        new_b = int(color._r * (100 + amount.value) / 100)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    raise ValueError('Cannot lighten by value:', str(amount))

builtin_funcs = {
    'darken': darken,
    'lighten': lighten,
    }
<commit_msg>use decorators for builtin functions instead of manually adding to dict<commit_after>from decimal import Decimal
from .types import Color, Distance

builtin_funcs = {}

### HELPER FUNCTIONS ###

def bif(f):
    'Decorator to build builtin_funcs dict.'
    builtin_funcs[f.__name__] = f

@bif
def darken(color, amount):
    if not isinstance(color, Color):
        raise ValueError('Cannot darken non-color:', str(color))

    if isinstance(amount, Decimal):
        new_r = int(color._r - amount)
        new_g = int(color._g - amount)
        new_b = int(color._b - amount)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Color):
        new_r = int(color._r - amount._r)
        new_g = int(color._g - amount._g)
        new_b = int(color._b - amount._b)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Distance) and amount.unit == '%':
        new_r = int(color._r * (100 - amount.value) / 100)
        new_g = int(color._r * (100 - amount.value) / 100)
        new_b = int(color._r * (100 - amount.value) / 100)
        color._r, color._g, color._b = (color if color >= 0 else 0
                                        for color in (new_r, new_g, new_b))
        return color

    raise ValueError('Cannot darken by value:', str(amount))

@bif
def lighten(color, amount):
    if not isinstance(color, Color):
        raise ValueError('Cannot lighten non-color:', str(color))

    if isinstance(amount, Decimal):
        new_r = int(color._r + amount)
        new_g = int(color._g + amount)
        new_b = int(color._b + amount)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Color):
        new_r = int(color._r + amount._r)
        new_g = int(color._g + amount._g)
        new_b = int(color._b + amount._b)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    elif isinstance(amount, Distance) and amount.unit == '%':
        new_r = int(color._r * (100 + amount.value) / 100)
        new_g = int(color._r * (100 + amount.value) / 100)
        new_b = int(color._r * (100 + amount.value) / 100)
        color._r, color._g, color._b = (color if color <= 255 else 255
                                        for color in (new_r, new_g, new_b))
        return color

    raise ValueError('Cannot lighten by value:', str(amount))
<|endoftext|>