<commit_before># coding=utf-8
# Copyright 2021 The Robustness Metrics Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3

r"""Reports that evaluate models over the ImageNet label space.

"""

import collections
from typing import Dict

from absl import logging
import numpy as np
from robustness_metrics.common import registry
from robustness_metrics.reports import base

# The numbers are from Appendix C in
#   "Benchmarking Neural Network Robustness to Common Corruptions and Surface
#    Variations", Hendrycks and Diettrich, ICLR 2019
ALEXENET_AVERAGE_ERRORS = {
    "gaussian_noise": 0.886,
    "shot_noise": 0.894,
    "impulse_noise": 0.923,
    "defocus_blur": 0.820,
    "glass_blur": 0.826,
    "motion_blur": 0.786,
    "zoom_blur": 0.798,
    "snow": 0.867,
    "frost": 0.827,
    "fog": 0.819,
    "brightness": 0.565,
    "contrast": 0.853,
    "elastic_transform": 0.646,
    "pixelate": 0.718,
    "jpeg_compression": 0.607,
}
ALEXNET_AVERAGE_ERR_CLEAN = 0.4348
IMAGENET_C_CORRUPTIONS = [
    "gaussian_noise", "shot_noise", "impulse_noise", "defocus_blur",
    "glass_blur", "motion_blur", "zoom_blur", "snow", "frost", "fog",
    "brightness", "contrast", "elastic_transform", "pixelate",
    "jpeg_compression",
]

@base.registry.register("imagenet_variants")
class ImagenetVariantsReport(base.Report):
  """Aggregated statistics over the ImageNet variants.

  This report contains the following ImageNet variants:
    * imagenet
    * imagenet_a,
    * imagenet_v2 (all variants)
    * imagenet_c (all variants)
  For each dataset, we compute accuracy, expected calibration
  error (ece), log-likelihood, Brier, timing, and adaptive ECE.
  """

  def __init__(self):
    super().__init__()
    self._corruption_metrics = collections.defaultdict(list)
    self._accuracy_per_corruption = collections.defaultdict(list)
    self._results = {}

  def _yield_metrics_to_evaluate(self, use_dataset_labelset=None):
    """Yields metrics to be evaluated."""
    metrics = ["accuracy", "nll", "brier"]
    if use_dataset_labelset is not None:
      metrics = [f"{metric}(use_dataset_labelset={use_dataset_labelset})"
                 for metric in metrics]
    yield from metrics

  def _yield_classification_specs(self, dataset, use_dataset_labelset=None):
    """Yields a MeasurementSpec for each metric and a given dataset."""
    for metric in self._yield_metrics_to_evaluate(
        use_dataset_labelset=use_dataset_labelset):
      yield base.MeasurementSpec(dataset, metric)

  @property
  def required_measurements(self):

    yield from self._yield_classification_specs("imagenet")
    yield from self._yield_classification_specs("imagenet_a")
    yield from self._yield_classification_specs("imagenet_r")

    severities = list(range(1, 6))
    for corruption_type in IMAGENET_C_CORRUPTIONS:
      for severity in severities:
        dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                   f"severity={severity})")
        yield from self._yield_classification_specs(dataset)
    yield from self._yield_classification_specs(
        "imagenet_v2(variant='MATCHED_FREQUENCY')")

  def add_measurement(self, dataset_spec, metric_name, metric_results):
    dataset_name, _, dataset_kwargs = registry.parse_name_and_kwargs(
        dataset_spec)
    if dataset_name == "imagenet_c":
      if metric_name == "timing":
        value = metric_results["mean"]
      else:
        # All remaining metrics are a dictionary with a single value, with
        # the key equal to the metric name.
        value, = list(metric_results.values())
      self._corruption_metrics[f"imagenet_c/{metric_name}"].append(value)
      corruption_type = dataset_kwargs["corruption_type"]
      if metric_name == "accuracy":
        self._accuracy_per_corruption[corruption_type].append(value)
    elif dataset_name == "imagenet_v2":
      variant = dataset_kwargs["variant"]
      if variant == "MATCHED_FREQUENCY":
        if metric_name == "timing":
          value = metric_results["mean"]
        else:
          # All remaining metrics are a dictionary with a single value, with
          # the key equal to the metric name.
          value, = list(metric_results.values())
        self._results[f"imagenet_v2/{metric_name}"] = value
      else:
        logging.info("Ignoring v2 variant %r", variant)
    else:
      if len(metric_results) == 1:
        # Metrics produce a dictionary with a single value, with the key equal
        # to the metric name.
        value, = list(metric_results.values())
      elif metric_name == "timing":
        value = metric_results["mean"]
      elif metric_name == "beta_scaling":
        value = 0.0  # Dummy.
      else:
        raise ValueError(
            f"Must specify which key to use: {list(metric_results.keys())}")
      self._results[f"{dataset_name}/{metric_name}"] = value

  def result(self) -> Dict[str, float]:
    # TODO(josipd): Check if all measurements are in.
    results = dict(self._results)
    results.update(base.compute_stats_per_bucket(self._corruption_metrics))
    ratios = []
    relative_ratios = []
    # We need errors instead of accuracies, so error = 1 - accuracy.
    try:
      clean_err = 1 - self._results["imagenet/accuracy"]
    except KeyError:
      raise base.MeasurementMissingError("Missing measurement: imagenet")
    for corruption in IMAGENET_C_CORRUPTIONS:
      if len(self._accuracy_per_corruption[corruption]) != 5:
        raise base.MeasurementMissingError(
            f"Not enough measurements for imagenet_c({corruption})")
      average_err = 1 - np.mean(
          self._accuracy_per_corruption[corruption])
      alexnet_average_err = ALEXENET_AVERAGE_ERRORS[corruption]
      ratios.append(average_err / alexnet_average_err)
      results[f"imagenet_c/{corruption}/ce"] = ratios[-1]
      relative_ratios.append((average_err - clean_err) /
                             (alexnet_average_err - ALEXNET_AVERAGE_ERR_CLEAN))
      results[f"imagenet_c/{corruption}/relative_ce"] = relative_ratios[-1]
    results["imagenet_c/mce"] = np.mean(ratios)
    results["imagenet_c/relative_mce"] = np.mean(relative_ratios)
    return results

@base.registry.register("imagenet_variants_gce_sweep")
class ImagenetVariantsGceSweepReport(ImagenetVariantsReport):
  """Sweeps over additional GCE variants."""

  def _yield_gce_metrics(self, use_dataset_labelset=None):
    # Add a sweep over General Calibration Errors:
    max_prob = True  # Whether to consider the max prediction or all.
    class_conditional = False  # Wheter to consider classes individually.
    threshold = 0.0  # Predicted probablities less than `threshold` are removed.
    recalibration_method = None  # Post-hoc recalibration on full dataset.
    if use_dataset_labelset is None:
      labelset_str = ""
    else:
      labelset_str = f"use_dataset_labelset={use_dataset_labelset},"
    for norm in ["l1", "l2"]:  # Which distance to use for the CE computation.
      for binning_scheme in ["adaptive", "even"]:
        for num_bins in [15, 100]:
          yield (f"gce("
                 f"binning_scheme={binning_scheme!r},"
                 f"max_prob={max_prob},"
                 f"class_conditional={class_conditional},"
                 f"norm='{norm}',"
                 f"num_bins={num_bins},"
                 f"threshold={threshold},"
                 f"recalibration_method={recalibration_method},"
                 f"{labelset_str}"
                 f")")

  def _yield_metrics_to_evaluate(self, use_dataset_labelset=None):
    """Yields metrics to be evaluated."""

    # Include standard metrics:
    yield from super()._yield_metrics_to_evaluate(
        use_dataset_labelset=use_dataset_labelset)
    yield from self._yield_gce_metrics(
        use_dataset_labelset=use_dataset_labelset)

  @property
  def required_measurements(self):
    yield from super().required_measurements
    yield base.MeasurementSpec("objectnet", "objectnet_accuracy")
    for gce_spec in self._yield_gce_metrics():
      yield base.MeasurementSpec("objectnet", f"objectnet_{gce_spec}")

@base.registry.register("imagenet_rescaling")
class ImagenetRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for ImageNet."""

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling", "beta_scaling"]
    for rescaling_method in rescaling_methods:
      metric = f"{rescaling_method}"
      yield base.MeasurementSpec("imagenet(split='validation[:20%]')", metric)
      yield base.MeasurementSpec("imagenet(split='validation[20%:]')", metric)

@base.registry.register("imagenet_c_rescaling")
class ImagenetCRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for ImageNet-C.

  This is not used for actual rescaling, but for over/under-confidence analysis.
  """

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling"]
    severities = list(range(1, 6))
    for rescaling_method in rescaling_methods:
      metric = f"{rescaling_method}"
      for corruption_type in IMAGENET_C_CORRUPTIONS:
        for severity in severities:
          dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                     f"severity={severity},"
                     "split='validation[:20%]')")
          dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                     f"severity={severity},"
                     "split='validation[20%:]')")
          yield base.MeasurementSpec(dataset, metric)

@base.registry.register("ood_rescaling")
class OodRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for OOD datasets.

  This is not used for actual rescaling, but for over/under-confidence analysis.
  """

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling"]
    for rescaling_method in rescaling_methods:
      yield base.MeasurementSpec("imagenet_a", rescaling_method)
      yield base.MeasurementSpec("imagenet_r", rescaling_method)
      yield base.MeasurementSpec(
          "imagenet_v2(variant='MATCHED_FREQUENCY')", rescaling_method)

@base.registry.register("imagenet_rescaled_variants_gce_sweep")
class ImagenetVariantsRescaledGceSweepReport(ImagenetVariantsGceSweepReport):
  """Similar to ImagenetVariantsGceSweepReport, but with [20%:] data split."""

  @property
  def required_measurements(self):
    split = "validation[20%:]"

    yield from self._yield_classification_specs(f"imagenet(split={split!r})")
    yield from self._yield_classification_specs(
        "imagenet_a", use_dataset_labelset=True)
    yield from self._yield_classification_specs(
        "imagenet_r", use_dataset_labelset=True)

    severities = list(range(1, 6))
    for corruption_type in IMAGENET_C_CORRUPTIONS:
      for severity in severities:
        dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                   f"severity={severity},"
                   f"split={split!r})")
        yield from self._yield_classification_specs(dataset)
    yield from self._yield_classification_specs(
        "imagenet_v2(variant='MATCHED_FREQUENCY')")
<commit_msg>Internal change<commit_after># coding=utf-8
# Copyright 2021 The Robustness Metrics Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3

r"""Reports that evaluate models over the ImageNet label space.

"""

import collections
from typing import Dict

from absl import logging
import numpy as np
from robustness_metrics.common import registry
from robustness_metrics.reports import base

# The numbers are from Appendix C in
#   "Benchmarking Neural Network Robustness to Common Corruptions and Surface
#    Variations", Hendrycks and Diettrich, ICLR 2019
ALEXENET_AVERAGE_ERRORS = {
    "gaussian_noise": 0.886,
    "shot_noise": 0.894,
    "impulse_noise": 0.923,
    "defocus_blur": 0.820,
    "glass_blur": 0.826,
    "motion_blur": 0.786,
    "zoom_blur": 0.798,
    "snow": 0.867,
    "frost": 0.827,
    "fog": 0.819,
    "brightness": 0.565,
    "contrast": 0.853,
    "elastic_transform": 0.646,
    "pixelate": 0.718,
    "jpeg_compression": 0.607,
}
ALEXNET_AVERAGE_ERR_CLEAN = 0.4348
IMAGENET_C_CORRUPTIONS = [
    "gaussian_noise", "shot_noise", "impulse_noise", "defocus_blur",
    "glass_blur", "motion_blur", "zoom_blur", "snow", "frost", "fog",
    "brightness", "contrast", "elastic_transform", "pixelate",
    "jpeg_compression",
]

@base.registry.register("imagenet_variants")
class ImagenetVariantsReport(base.Report):
  """Aggregated statistics over the ImageNet variants.

  This report contains the following ImageNet variants:
    * imagenet
    * imagenet_a,
    * imagenet_v2 (all variants)
    * imagenet_c (all variants)
  For each dataset, we compute accuracy, expected calibration
  error (ece), log-likelihood, Brier, timing, and adaptive ECE.
  """

  def __init__(self):
    super().__init__()
    self._corruption_metrics = collections.defaultdict(list)
    self._accuracy_per_corruption = collections.defaultdict(list)
    self._results = {}

  def _yield_metrics_to_evaluate(self, use_dataset_labelset=None):
    """Yields metrics to be evaluated."""
    metrics = ["accuracy", "nll", "brier"]
    if use_dataset_labelset is not None:
      metrics = [f"{metric}(use_dataset_labelset={use_dataset_labelset})"
                 for metric in metrics]
    yield from metrics

  def _yield_classification_specs(self, dataset, use_dataset_labelset=None):
    """Yields a MeasurementSpec for each metric and a given dataset."""
    for metric in self._yield_metrics_to_evaluate(
        use_dataset_labelset=use_dataset_labelset):
      yield base.MeasurementSpec(dataset, metric)

  @property
  def required_measurements(self):

    yield from self._yield_classification_specs("imagenet")
    yield from self._yield_classification_specs("imagenet_a")
    yield from self._yield_classification_specs("imagenet_r")

    severities = list(range(1, 6))
    for corruption_type in IMAGENET_C_CORRUPTIONS:
      for severity in severities:
        dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                   f"severity={severity})")
        yield from self._yield_classification_specs(dataset)
    yield from self._yield_classification_specs(
        "imagenet_v2(variant='MATCHED_FREQUENCY')")

  def add_measurement(self, dataset_spec, metric_name, metric_results):
    dataset_name, _, dataset_kwargs = registry.parse_name_and_kwargs(
        dataset_spec)
    if dataset_name == "imagenet_c":
      if metric_name == "timing":
        value = metric_results["mean"]
      else:
        # All remaining metrics are a dictionary with a single value, with
        # the key equal to the metric name.
        value, = list(metric_results.values())
      self._corruption_metrics[f"imagenet_c/{metric_name}"].append(value)
      corruption_type = dataset_kwargs["corruption_type"]
      if metric_name == "accuracy":
        self._accuracy_per_corruption[corruption_type].append(value)
    elif dataset_name == "imagenet_v2":
      variant = dataset_kwargs["variant"]
      if variant == "MATCHED_FREQUENCY":
        if metric_name == "timing":
          value = metric_results["mean"]
        else:
          # All remaining metrics are a dictionary with a single value, with
          # the key equal to the metric name.
          value, = list(metric_results.values())
        self._results[f"imagenet_v2/{metric_name}"] = value
      else:
        logging.info("Ignoring v2 variant %r", variant)
    else:
      if len(metric_results) == 1:
        # Metrics produce a dictionary with a single value, with the key equal
        # to the metric name.
        value, = list(metric_results.values())
      elif metric_name == "timing":
        value = metric_results["mean"]
      else:
        raise ValueError(
            f"Must specify which key to use: {list(metric_results.keys())}")
      self._results[f"{dataset_name}/{metric_name}"] = value

  def result(self) -> Dict[str, float]:
    # TODO(josipd): Check if all measurements are in.
    results = dict(self._results)
    results.update(base.compute_stats_per_bucket(self._corruption_metrics))
    ratios = []
    relative_ratios = []
    # We need errors instead of accuracies, so error = 1 - accuracy.
    try:
      clean_err = 1 - self._results["imagenet/accuracy"]
    except KeyError:
      raise base.MeasurementMissingError("Missing measurement: imagenet")
    for corruption in IMAGENET_C_CORRUPTIONS:
      if len(self._accuracy_per_corruption[corruption]) != 5:
        raise base.MeasurementMissingError(
            f"Not enough measurements for imagenet_c({corruption})")
      average_err = 1 - np.mean(
          self._accuracy_per_corruption[corruption])
      alexnet_average_err = ALEXENET_AVERAGE_ERRORS[corruption]
      ratios.append(average_err / alexnet_average_err)
      results[f"imagenet_c/{corruption}/ce"] = ratios[-1]
      relative_ratios.append((average_err - clean_err) /
                             (alexnet_average_err - ALEXNET_AVERAGE_ERR_CLEAN))
      results[f"imagenet_c/{corruption}/relative_ce"] = relative_ratios[-1]
    results["imagenet_c/mce"] = np.mean(ratios)
    results["imagenet_c/relative_mce"] = np.mean(relative_ratios)
    return results

@base.registry.register("imagenet_variants_gce_sweep")
class ImagenetVariantsGceSweepReport(ImagenetVariantsReport):
  """Sweeps over additional GCE variants."""

  def _yield_gce_metrics(self, use_dataset_labelset=None):
    # Add a sweep over General Calibration Errors:
    max_prob = True  # Whether to consider the max prediction or all.
    class_conditional = False  # Wheter to consider classes individually.
    threshold = 0.0  # Predicted probablities less than `threshold` are removed.
    recalibration_method = None  # Post-hoc recalibration on full dataset.
    if use_dataset_labelset is None:
      labelset_str = ""
    else:
      labelset_str = f"use_dataset_labelset={use_dataset_labelset},"
    for norm in ["l1", "l2"]:  # Which distance to use for the CE computation.
      for binning_scheme in ["adaptive", "even"]:
        for num_bins in [15, 100]:
          yield (f"gce("
                 f"binning_scheme={binning_scheme!r},"
                 f"max_prob={max_prob},"
                 f"class_conditional={class_conditional},"
                 f"norm='{norm}',"
                 f"num_bins={num_bins},"
                 f"threshold={threshold},"
                 f"recalibration_method={recalibration_method},"
                 f"{labelset_str}"
                 f")")

  def _yield_metrics_to_evaluate(self, use_dataset_labelset=None):
    """Yields metrics to be evaluated."""

    # Include standard metrics:
    yield from super()._yield_metrics_to_evaluate(
        use_dataset_labelset=use_dataset_labelset)
    yield from self._yield_gce_metrics(
        use_dataset_labelset=use_dataset_labelset)

  @property
  def required_measurements(self):
    yield from super().required_measurements
    yield base.MeasurementSpec("objectnet", "objectnet_accuracy")
    for gce_spec in self._yield_gce_metrics():
      yield base.MeasurementSpec("objectnet", f"objectnet_{gce_spec}")

@base.registry.register("imagenet_rescaling")
class ImagenetRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for ImageNet."""

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling"]
    for rescaling_method in rescaling_methods:
      metric = f"{rescaling_method}"
      yield base.MeasurementSpec("imagenet(split='validation[:20%]')", metric)
      yield base.MeasurementSpec("imagenet(split='validation[20%:]')", metric)

@base.registry.register("imagenet_c_rescaling")
class ImagenetCRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for ImageNet-C.

  This is not used for actual rescaling, but for over/under-confidence analysis.
  """

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling"]
    severities = list(range(1, 6))
    for rescaling_method in rescaling_methods:
      metric = f"{rescaling_method}"
      for corruption_type in IMAGENET_C_CORRUPTIONS:
        for severity in severities:
          dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                     f"severity={severity},"
                     "split='validation[:20%]')")
          dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                     f"severity={severity},"
                     "split='validation[20%:]')")
          yield base.MeasurementSpec(dataset, metric)

@base.registry.register("ood_rescaling")
class OodRescalingReport(ImagenetVariantsReport):
  """Report that computes rescaling parameters for OOD datasets.

  This is not used for actual rescaling, but for over/under-confidence analysis.
  """

  @property
  def required_measurements(self):
    rescaling_methods = ["temperature_scaling"]
    for rescaling_method in rescaling_methods:
      yield base.MeasurementSpec("imagenet_a", rescaling_method)
      yield base.MeasurementSpec("imagenet_r", rescaling_method)
      yield base.MeasurementSpec(
          "imagenet_v2(variant='MATCHED_FREQUENCY')", rescaling_method)

@base.registry.register("imagenet_rescaled_variants_gce_sweep")
class ImagenetVariantsRescaledGceSweepReport(ImagenetVariantsGceSweepReport):
  """Similar to ImagenetVariantsGceSweepReport, but with [20%:] data split."""

  @property
  def required_measurements(self):
    split = "validation[20%:]"

    yield from self._yield_classification_specs(f"imagenet(split={split!r})")
    yield from self._yield_classification_specs(
        "imagenet_a", use_dataset_labelset=True)
    yield from self._yield_classification_specs(
        "imagenet_r", use_dataset_labelset=True)

    severities = list(range(1, 6))
    for corruption_type in IMAGENET_C_CORRUPTIONS:
      for severity in severities:
        dataset = (f"imagenet_c(corruption_type={corruption_type!r},"
                   f"severity={severity},"
                   f"split={split!r})")
        yield from self._yield_classification_specs(dataset)
    yield from self._yield_classification_specs(
        "imagenet_v2(variant='MATCHED_FREQUENCY')")
<|endoftext|>


<commit_before>#
# developer.py
#
# developer functions for managing SusumuTakuan remotely
#

import discord
import asyncio
import subprocess

from access import has_access, grant_role_access
from database import Command, CommandClass, Role, User, get_user_by_id

#Create database registration function
def register_functions(session):

	class_id = session.query(CommandClass).filter(CommandClass.name == 'developer').first()
	if ( class_id == None ):
		dev_class = CommandClass(name="developer")
		session.add(dev_class)

	cmd_id = session.query(Command).filter(Command.name == 'update_git').first()
	if (cmd_id == None ):
		cmd_update_git = Command(name="update_git")
		cmd_update_git.command_class = dev_class
		session.add(cmd_update_git)

	cmd_id = session.query(Command).filter(Command.name == 'restart_bot').first()
	if (cmd_id == None ):
		cmd_restart_bot = Command(name="restart_bot")
		cmd_restart_bot.command_class = dev_class
		session.add(cmd_restart_bot)

	cmd_id = session.query(Command).filter(Command.name == 'debug_output').first()
	if (cmd_id == None ):
		cmd_debug_output = Command(name="debug_output")
		cmd_debug_output.command_class = dev_class
		session.add(cmd_debug_output)

	cmd_id = session.query(Command).filter(Command.name == 'debug_error').first()
	if (cmd_id == None ):
		cmd_debug_error = Command(name="debug_error")
		cmd_debug_error.command_class = dev_class
		session.add(cmd_debug_error)

	session.commit()

#Add developers to internal group
def register_developer_access(session, developers):

	developer_class = session.query(CommandClass).filter(CommandClass.name == 'developer').first()
	developer_role = session.query(Role).filter(Role.name == 'developer', Role.server_id == 1).first()

	grant_role_access(session, developer_role, developer_class)

	for developer in developers:
		the_developer = session.query(User).filter(User.id == developer).first()
		if ( the_developer != None):
			the_developer.roles.append(developer_role)

	session.commit()

async def update_git(client, message, session):
	command_class = session.query(Command).filter(Command.name == 'update_git').first()
	tmp = await client.send_message(message.channel, 'Updating my code via git...')
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested to update my code.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command_class)):
			process = subprocess.run(["sh", "control.sh", "refresh"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run update command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def restart_bot(client, message, session):
	command_class = session.query(Command).filter(Command.name == 'restart_bot').first()
	tmp = await client.send_message(message.channel, 'Restarting myself...')
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested to restart me.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command_class)):
			process = subprocess.run(["sh", "control.sh", "restart"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run restart command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def debug_output(client, message, session):
	command_class = session.query(Command).filter(Command.name == 'debug_output').first()
	tmp = await client.send_message(message.channel, 'Providing debug log of stdout...')
	message_array=message.content.split(" ")
	try:
		num_lines=int(message_array[1])
	except ValueError:
		print("debug_error: User gave invalid value for number of lines")
		tmp = await client.send_message(message.channel, '%s is not a valid number of lines' % (message_array[1]))
	log_lines='-%d' % (num_lines)
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested output log.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command_class)):
			process = subprocess.run(["tail", log_lines, "logs/output.log"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run debug command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def debug_error(client, message, session):
	command_class = session.query(Command).filter(Command.name == 'debug_error').first()
	tmp = await client.send_message(message.channel, 'Providing debug log of stderr...')
	message_array=message.content.split(" ")
	try:
		num_lines=int(message_array[1])
	except ValueError:
		print("debug_error: User gave invalid value for number of lines")
		tmp = await client.send_message(message.channel, '%s is not a valid number of lines' % (message_array[1]))
	log_lines='-%d' % (num_lines)
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested error log.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command_class)):
			process = subprocess.run(["tail", log_lines, "logs/error.log"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run debug command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')<commit_msg>Pass in command_class, not command<commit_after>#
# developer.py
#
# developer functions for managing SusumuTakuan remotely
#

import discord
import asyncio
import subprocess

from access import has_access, grant_role_access
from database import Command, CommandClass, Role, User, get_user_by_id

#Create database registration function
def register_functions(session):

	class_id = session.query(CommandClass).filter(CommandClass.name == 'developer').first()
	if ( class_id == None ):
		dev_class = CommandClass(name="developer")
		session.add(dev_class)

	cmd_id = session.query(Command).filter(Command.name == 'update_git').first()
	if (cmd_id == None ):
		cmd_update_git = Command(name="update_git")
		cmd_update_git.command_class = dev_class
		session.add(cmd_update_git)

	cmd_id = session.query(Command).filter(Command.name == 'restart_bot').first()
	if (cmd_id == None ):
		cmd_restart_bot = Command(name="restart_bot")
		cmd_restart_bot.command_class = dev_class
		session.add(cmd_restart_bot)

	cmd_id = session.query(Command).filter(Command.name == 'debug_output').first()
	if (cmd_id == None ):
		cmd_debug_output = Command(name="debug_output")
		cmd_debug_output.command_class = dev_class
		session.add(cmd_debug_output)

	cmd_id = session.query(Command).filter(Command.name == 'debug_error').first()
	if (cmd_id == None ):
		cmd_debug_error = Command(name="debug_error")
		cmd_debug_error.command_class = dev_class
		session.add(cmd_debug_error)

	session.commit()

#Add developers to internal group
def register_developer_access(session, developers):

	developer_class = session.query(CommandClass).filter(CommandClass.name == 'developer').first()
	developer_role = session.query(Role).filter(Role.name == 'developer', Role.server_id == 1).first()

	grant_role_access(session, developer_role, developer_class)

	for developer in developers:
		the_developer = session.query(User).filter(User.id == developer).first()
		if ( the_developer != None):
			the_developer.roles.append(developer_role)

	session.commit()

async def update_git(client, message, session):
	command = session.query(Command).filter(Command.name == 'update_git').first()
	tmp = await client.send_message(message.channel, 'Updating my code via git...')
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested to update my code.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command.command_class)):
			process = subprocess.run(["sh", "control.sh", "refresh"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run update command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def restart_bot(client, message, session):
	command = session.query(Command).filter(Command.name == 'restart_bot').first()
	tmp = await client.send_message(message.channel, 'Restarting myself...')
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested to restart me.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command.command_class)):
			process = subprocess.run(["sh", "control.sh", "restart"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run restart command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def debug_output(client, message, session):
	command = session.query(Command).filter(Command.name == 'debug_output').first()
	tmp = await client.send_message(message.channel, 'Providing debug log of stdout...')
	message_array=message.content.split(" ")
	try:
		num_lines=int(message_array[1])
	except ValueError:
		print("debug_error: User gave invalid value for number of lines")
		tmp = await client.send_message(message.channel, '%s is not a valid number of lines' % (message_array[1]))
	log_lines='-%d' % (num_lines)
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested output log.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command.command_class)):
			process = subprocess.run(["tail", log_lines, "logs/output.log"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run debug command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')

async def debug_error(client, message, session):
	command = session.query(Command).filter(Command.name == 'debug_error').first()
	tmp = await client.send_message(message.channel, 'Providing debug log of stderr...')
	message_array=message.content.split(" ")
	try:
		num_lines=int(message_array[1])
	except ValueError:
		print("debug_error: User gave invalid value for number of lines")
		tmp = await client.send_message(message.channel, '%s is not a valid number of lines' % (message_array[1]))
	log_lines='-%d' % (num_lines)
	users = message.channel.recipients
	for user in users:
		if user.id != client.user.id:
			print('%s/%s requested error log.' % (user.name, user.id))

		db_user = get_user_by_id(session, user.id)
		if ( has_access(session, db_user, command.command_class)):
			process = subprocess.run(["tail", log_lines, "logs/error.log"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			tmp = await client.send_message(message.channel, process.stdout)
		else:
			print('%s/%s not allowed to run debug command.' % (user.name, user.id))
			tmp = await client.send_message(message.channel, 'Unauthorized')<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#=======================================================================
#
# chacha_test.py
# --------------
# Simple model of the ChaCha cipher as specified in RFC 7539
# (https://tools.ietf.org/html/rfc7539).
#
#
# Copyright (c) 2016 Secworks Sweden AB
# Author: Joachim Strömbergson
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#=======================================================================

#-------------------------------------------------------------------
# Python module imports.
#-------------------------------------------------------------------
import sys

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Helper functions.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# print_chacha_state()
#
# Print the given chacha state matrix.
#-------------------------------------------------------------------
def print_chacha_state(state):
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[00], state[1],  state[2],  state[3]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[4],  state[5],  state[6],  state[7]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[8],  state[9],  state[10], state[11]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[12], state[13], state[14], state[15]))
    print("")

#-------------------------------------------------------------------
# check_chacha_state()
#
# Check a givem chacha state against the given expected state.
# Report if state is correct, or which elements are incorrect.
#-------------------------------------------------------------------
def check_chacha_state(state, expected):
    errors = 0
    for i in range(len(state)):
        if state[i] != expected[i]:
            print("state[%02d] = 0x%08x does not match expected 0x%08x" %
                      (i, state[i], expected[i]))
            errors += 1

    if (errors > 0):
        print("State is incorrect at %02d elements" % errors)
    else:
        print("State is correct.")
    print("")

#-------------------------------------------------------------------
# rotl()
#
# Rotate 32-bit operand giveb number of bits left.
#-------------------------------------------------------------------
def rotl(op, bits):
    assert bits < 33
    return ((op << bits) + (op >> (32 - bits))) & 0xffffffff

#-------------------------------------------------------------------
# l2lw32()
#
# Convert a given list of bytes to list of little endian
# 32-bit endian words.
#-------------------------------------------------------------------
def l2lw32(bytelist):
    num_words = int(len(bytelist) / 4)
    chunks = [bytelist[(i * 4) : (i*4 + 4)] for i in range(num_words)]
    return [((b[3] << 24) + (b[2] << 16) + (b[1] << 8)) for b in chunks]

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Tested functions.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# qr()
#
# The ChaCha qr function.
#-------------------------------------------------------------------
def qr(a, b, c, d):
    a0 = (a + b) & 0xffffffff
    d0 = d ^ a0
    d1 = rotl(d0, 16)
    c0 = (c + d1) & 0xffffffff
    b0 = b ^ c0
    b1 = rotl(b0, 12)
    a1 = (a0 + b1) & 0xffffffff
    d2 = d1 ^ a1
    d3 = rotl(d2, 8)
    c1 = (c0 + d3) & 0xffffffff
    b2 = b1 ^ c1
    b3 = rotl(b2, 7)
    return (a1, b3, c1, d3)

#-------------------------------------------------------------------
# quarterround()
#
# Update the given state by applying the qr function on the
# given elements in the state.
#-------------------------------------------------------------------
def quarterround(state, ai, bi, ci, di):
    (ap, bp, cp, dp) = qr(state[ai], state[bi], state[ci], state[di])
    state[ai] = ap
    state[bi] = bp
    state[ci] = cp
    state[di] = dp
    return state

#-------------------------------------------------------------------
# doubleround()
#
# Perform the ChaCha doubleround on the given state by applying
# eigth specific quarterrounds.
#-------------------------------------------------------------------
def doubleround(state):
    quarterround(state, 0, 4, 8,12)
    quarterround(state, 1, 5, 9,13)
    quarterround(state, 2, 6,10,14)
    quarterround(state, 3, 7,11,15)
    quarterround(state, 0, 5,10,15)
    quarterround(state, 1, 6,11,12)
    quarterround(state, 2, 7, 8,13)
    quarterround(state, 3, 4, 9,14)
    return state

#-------------------------------------------------------------------
# chacha_block()
#
# The chacha block function. Given a 256 bit key, 32 bit counter
# and 96 bit nonce will create a state and then update the state
# for 10 doublerounds. Finally the finalized state is returned
# as a sequence of bytes.
#
# This code follows the pseudo code in 2.3.1 in RFC 7539.
#-------------------------------------------------------------------
def chacha_block(key, counter, nonce):
    pass

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Tests.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# run_rotl_test()
#-------------------------------------------------------------------
def run_rotl_test():
    print("Test of rotl function.")
    for i in range(32):
        print("%02d: 0x%08x" % (i, rotl(0x10000001, i)))
    print("")

#-------------------------------------------------------------------
# run_qr_test()
#
# Test qr function with test vector in chapter 2.1.1.
#-------------------------------------------------------------------
def run_qr_test():
    a = 0x11111111
    b = 0x01020304
    c = 0x9b8d6f43
    d = 0x01234567
    (ap, bp, cp, dp) = qr(a, b, c, d)
    print("Test of qr function:")
    print("a:  0x%08x b:  0x%08x c:  0x%08x d:  0x%08x" % (a, b, c, d))
    print("ap: 0x%08x bp: 0x%08x cp: 0x%08x dp: 0x%08x" % (ap, bp, cp, dp))

    if ((ap != 0xea2a92f4) or (bp != 0xcb1cf8ce) or
        (cp != 0x4581472e) or (dp != 0x5881c4bb)):
        print("Incorrect result generated.")
    else:
        print("Correct result generated.")
    print("")

#-------------------------------------------------------------------
# run_qr_chacha_state_test()
#
# Test the quarterround function applied to the chacha state.
# Test vectors from chapter 2.2.1 in RFC 7539.
#-------------------------------------------------------------------
def run_qr_chacha_state_test():
    chacha_state = [0] * 16

    init_state = [0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a,
                  0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c,
                  0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963,
                  0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320]

    expected_state = [0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a,
                      0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2,
                      0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963,
                      0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320]

    print("ChaCha quarterround update test.")
    print("ChaCha state after init:")
    for i in range(len(init_state)):
        chacha_state[i] = init_state[i]
    print_chacha_state(chacha_state)

    print("ChaCha state after quarterround(2, 7, 8, 13):")
    chacha_state = quarterround(chacha_state, 2, 7, 8, 13)
    print_chacha_state(chacha_state)
    check_chacha_state(chacha_state, expected_state)

#-------------------------------------------------------------------
# run_chacha_doubleround_function_test()
#
# Test the chacha doubleround function applied 10 times
# on the chacha state.
# Test vectors from chapter 2.3.2 in RFC 7539.
#-------------------------------------------------------------------
def run_chacha_doubleround_function_test():
    init_state = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,
                  0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
                  0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
                  0x00000001, 0x09000000, 0x4a000000, 0x00000000]

    expected_state = [0x837778ab, 0xe238d763, 0xa67ae21e, 0x5950bb2f,
                      0xc4f2d0c7, 0xfc62bb2f, 0x8fa018fc, 0x3f5ec7b7,
                      0x335271c2, 0xf29489f3, 0xeabda8fc, 0x82e46ebd,
                      0xd19c12b4, 0xb04e16de, 0x9e83d0cb, 0x4e3c50a2]

    chacha_state = [0] * 16
    for i in range(len(init_state)):
        chacha_state[i] = init_state[i]
    print("ChaCha state after init:")
    print_chacha_state(chacha_state)

    print("ChaCha state updates for 10 doublerounds:")
    for i in range(10):
        chacha_state = doubleround(chacha_state)
        print("After round %02d:" % (i + 1))
        print_chacha_state(chacha_state)

    check_chacha_state(chacha_state, expected_state)

#-------------------------------------------------------------------
# run_chacha_block_test()
#-------------------------------------------------------------------
def run_chacha_block_test():
    key_bytes = [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
                 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
                 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
                 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f]

    key = l2lw32(key_bytes)
    print(key)

#-------------------------------------------------------------------
# main()
#
# Run chacha tests.
#-------------------------------------------------------------------
def main():
#    run_rotl_test()
#    run_qr_test()
#    run_qr_chacha_state_test()
#    run_chacha_doubleround_function_test()
    run_chacha_block_test()

#-------------------------------------------------------------------
#-------------------------------------------------------------------
if __name__=="__main__":
    sys.exit(main())

#=======================================================================
# EOF chacha_test.py
#=======================================================================
<commit_msg>Adding nonce bytes and convertionn to nonce words.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#=======================================================================
#
# chacha_test.py
# --------------
# Simple model of the ChaCha cipher as specified in RFC 7539
# (https://tools.ietf.org/html/rfc7539).
#
#
# Copyright (c) 2016 Secworks Sweden AB
# Author: Joachim Strömbergson
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#=======================================================================

#-------------------------------------------------------------------
# Python module imports.
#-------------------------------------------------------------------
import sys

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Helper functions.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# print_chacha_state()
#
# Print the given chacha state matrix.
#-------------------------------------------------------------------
def print_chacha_state(state):
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[00], state[1],  state[2],  state[3]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[4],  state[5],  state[6],  state[7]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[8],  state[9],  state[10], state[11]))
    print("0x%08x 0x%08x 0x%08x 0x%08x" % (state[12], state[13], state[14], state[15]))
    print("")

#-------------------------------------------------------------------
# check_chacha_state()
#
# Check a givem chacha state against the given expected state.
# Report if state is correct, or which elements are incorrect.
#-------------------------------------------------------------------
def check_chacha_state(state, expected):
    errors = 0
    for i in range(len(state)):
        if state[i] != expected[i]:
            print("state[%02d] = 0x%08x does not match expected 0x%08x" %
                      (i, state[i], expected[i]))
            errors += 1

    if (errors > 0):
        print("State is incorrect at %02d elements" % errors)
    else:
        print("State is correct.")
    print("")

#-------------------------------------------------------------------
# rotl()
#
# Rotate 32-bit operand giveb number of bits left.
#-------------------------------------------------------------------
def rotl(op, bits):
    assert bits < 33
    return ((op << bits) + (op >> (32 - bits))) & 0xffffffff

#-------------------------------------------------------------------
# l2lw32()
#
# Convert a given list of bytes to list of little endian
# 32-bit endian words.
#-------------------------------------------------------------------
def l2lw32(bytelist):
    num_words = int(len(bytelist) / 4)
    chunks = [bytelist[(i * 4) : (i*4 + 4)] for i in range(num_words)]
    return [((b[3] << 24) + (b[2] << 16) + (b[1] << 8)) for b in chunks]

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Tested functions.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# qr()
#
# The ChaCha qr function.
#-------------------------------------------------------------------
def qr(a, b, c, d):
    a0 = (a + b) & 0xffffffff
    d0 = d ^ a0
    d1 = rotl(d0, 16)
    c0 = (c + d1) & 0xffffffff
    b0 = b ^ c0
    b1 = rotl(b0, 12)
    a1 = (a0 + b1) & 0xffffffff
    d2 = d1 ^ a1
    d3 = rotl(d2, 8)
    c1 = (c0 + d3) & 0xffffffff
    b2 = b1 ^ c1
    b3 = rotl(b2, 7)
    return (a1, b3, c1, d3)

#-------------------------------------------------------------------
# quarterround()
#
# Update the given state by applying the qr function on the
# given elements in the state.
#-------------------------------------------------------------------
def quarterround(state, ai, bi, ci, di):
    (ap, bp, cp, dp) = qr(state[ai], state[bi], state[ci], state[di])
    state[ai] = ap
    state[bi] = bp
    state[ci] = cp
    state[di] = dp
    return state

#-------------------------------------------------------------------
# doubleround()
#
# Perform the ChaCha doubleround on the given state by applying
# eigth specific quarterrounds.
#-------------------------------------------------------------------
def doubleround(state):
    quarterround(state, 0, 4, 8,12)
    quarterround(state, 1, 5, 9,13)
    quarterround(state, 2, 6,10,14)
    quarterround(state, 3, 7,11,15)
    quarterround(state, 0, 5,10,15)
    quarterround(state, 1, 6,11,12)
    quarterround(state, 2, 7, 8,13)
    quarterround(state, 3, 4, 9,14)
    return state

#-------------------------------------------------------------------
# chacha_block()
#
# The chacha block function. Given a 256 bit key, 32 bit counter
# and 96 bit nonce will create a state and then update the state
# for 10 doublerounds. Finally the finalized state is returned
# as a sequence of bytes.
#
# This code follows the pseudo code in 2.3.1 in RFC 7539.
#-------------------------------------------------------------------
def chacha_block(key, counter, nonce):
    pass

#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Tests.
#-------------------------------------------------------------------
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# run_rotl_test()
#-------------------------------------------------------------------
def run_rotl_test():
    print("Test of rotl function.")
    for i in range(32):
        print("%02d: 0x%08x" % (i, rotl(0x10000001, i)))
    print("")

#-------------------------------------------------------------------
# run_qr_test()
#
# Test qr function with test vector in chapter 2.1.1.
#-------------------------------------------------------------------
def run_qr_test():
    a = 0x11111111
    b = 0x01020304
    c = 0x9b8d6f43
    d = 0x01234567
    (ap, bp, cp, dp) = qr(a, b, c, d)
    print("Test of qr function:")
    print("a:  0x%08x b:  0x%08x c:  0x%08x d:  0x%08x" % (a, b, c, d))
    print("ap: 0x%08x bp: 0x%08x cp: 0x%08x dp: 0x%08x" % (ap, bp, cp, dp))

    if ((ap != 0xea2a92f4) or (bp != 0xcb1cf8ce) or
        (cp != 0x4581472e) or (dp != 0x5881c4bb)):
        print("Incorrect result generated.")
    else:
        print("Correct result generated.")
    print("")

#-------------------------------------------------------------------
# run_qr_chacha_state_test()
#
# Test the quarterround function applied to the chacha state.
# Test vectors from chapter 2.2.1 in RFC 7539.
#-------------------------------------------------------------------
def run_qr_chacha_state_test():
    chacha_state = [0] * 16

    init_state = [0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a,
                  0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c,
                  0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963,
                  0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320]

    expected_state = [0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a,
                      0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2,
                      0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963,
                      0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320]

    print("ChaCha quarterround update test.")
    print("ChaCha state after init:")
    for i in range(len(init_state)):
        chacha_state[i] = init_state[i]
    print_chacha_state(chacha_state)

    print("ChaCha state after quarterround(2, 7, 8, 13):")
    chacha_state = quarterround(chacha_state, 2, 7, 8, 13)
    print_chacha_state(chacha_state)
    check_chacha_state(chacha_state, expected_state)

#-------------------------------------------------------------------
# run_chacha_doubleround_function_test()
#
# Test the chacha doubleround function applied 10 times
# on the chacha state.
# Test vectors from chapter 2.3.2 in RFC 7539.
#-------------------------------------------------------------------
def run_chacha_doubleround_function_test():
    init_state = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,
                  0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
                  0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
                  0x00000001, 0x09000000, 0x4a000000, 0x00000000]

    expected_state = [0x837778ab, 0xe238d763, 0xa67ae21e, 0x5950bb2f,
                      0xc4f2d0c7, 0xfc62bb2f, 0x8fa018fc, 0x3f5ec7b7,
                      0x335271c2, 0xf29489f3, 0xeabda8fc, 0x82e46ebd,
                      0xd19c12b4, 0xb04e16de, 0x9e83d0cb, 0x4e3c50a2]

    chacha_state = [0] * 16
    for i in range(len(init_state)):
        chacha_state[i] = init_state[i]
    print("ChaCha state after init:")
    print_chacha_state(chacha_state)

    print("ChaCha state updates for 10 doublerounds:")
    for i in range(10):
        chacha_state = doubleround(chacha_state)
        print("After round %02d:" % (i + 1))
        print_chacha_state(chacha_state)

    check_chacha_state(chacha_state, expected_state)

#-------------------------------------------------------------------
# run_chacha_block_test()
#-------------------------------------------------------------------
def run_chacha_block_test():
    key_bytes = [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
                 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
                 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
                 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f]

    key = l2lw32(key_bytes)
    print(key)

    nonce_bytes = [0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a,
                   0x00, 0x00, 0x00, 0x00]
    nonce = l2lw32(nonce_bytes)
    print(nonce)

#-------------------------------------------------------------------
# main()
#
# Run chacha tests.
#-------------------------------------------------------------------
def main():
#    run_rotl_test()
#    run_qr_test()
#    run_qr_chacha_state_test()
#    run_chacha_doubleround_function_test()
    run_chacha_block_test()

#-------------------------------------------------------------------
#-------------------------------------------------------------------
if __name__=="__main__":
    sys.exit(main())

#=======================================================================
# EOF chacha_test.py
#=======================================================================
<|endoftext|>


<commit_before>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

from robot.errors import DataError
from robot.parsing import VALID_EXTENSIONS

if sys.platform.startswith('java'):
    from .javalibdocbuilder import JavaDocBuilder
else:
    def JavaDocBuilder():
        raise DataError('Documenting Java test libraries requires Jython.')
from .librarydocoutput import LibraryDocOutput
from .robotlibdoc import LibraryDocBuilder, ResourceDocBuilder
from .speclibdocbuilder import SpecLibraryDocBuilder
from .xmlwriter import LibdocXmlWriter
from .libdochtmlwriter import LibdocHtmlWriter

def LibraryDoc(library_or_resource, arguments=None, name=None, version=None):
    builder = BuilderFactory(arguments, library_or_resource)
    libdoc = builder.build(library_or_resource, arguments)
    if name:
        libdoc.name = name
    if version:
        libdoc.version = version
    return libdoc

def BuilderFactory(arguments, library_or_resource):
    extension = os.path.splitext(library_or_resource)[1][1:].lower()
    if extension in VALID_EXTENSIONS:
        return ResourceDocBuilder()
    if extension == 'xml':
        return SpecLibraryDocBuilder()
    if extension == 'java':
        return JavaDocBuilder()
    return LibraryDocBuilder()

def LibraryDocWriter(format=None):
    format = (format or 'HTML').upper()
    if format == 'HTML':
        return LibdocHtmlWriter()
    if format == 'XML':
        return LibdocXmlWriter()
    raise DataError("Format must be either 'HTML' or 'XML', got '%s'." % format)
<commit_msg>libdoc cleanup<commit_after>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

from robot.errors import DataError
from robot.parsing import VALID_EXTENSIONS as RESOURCE_EXTENSIONS

if sys.platform.startswith('java'):
    from .javalibdocbuilder import JavaDocBuilder
else:
    def JavaDocBuilder():
        raise DataError('Documenting Java test libraries requires Jython.')
from .librarydocoutput import LibraryDocOutput
from .robotlibdoc import LibraryDocBuilder, ResourceDocBuilder
from .speclibdocbuilder import SpecLibraryDocBuilder
from .xmlwriter import LibdocXmlWriter
from .libdochtmlwriter import LibdocHtmlWriter

def LibraryDoc(library_or_resource, arguments=None, name=None, version=None):
    builder = BuilderFactory(library_or_resource)
    libdoc = builder.build(library_or_resource, arguments)
    if name:
        libdoc.name = name
    if version:
        libdoc.version = version
    return libdoc

def BuilderFactory(library_or_resource):
    extension = os.path.splitext(library_or_resource)[1][1:].lower()
    if extension in RESOURCE_EXTENSIONS:
        return ResourceDocBuilder()
    if extension == 'xml':
        return SpecLibraryDocBuilder()
    if extension == 'java':
        return JavaDocBuilder()
    return LibraryDocBuilder()

def LibraryDocWriter(format=None):
    format = (format or 'HTML').upper()
    if format == 'HTML':
        return LibdocHtmlWriter()
    if format == 'XML':
        return LibdocXmlWriter()
    raise DataError("Format must be either 'HTML' or 'XML', got '%s'." % format)
<|endoftext|>


<commit_before>"""
Name: Dimitri Somoff
Final Project: Flappy bird remake "Crappy Bird"
"""
from ggame import App, Color, LineStyle, Sprite, ImageAsset, Frame, SoundAsset
import random

wals = 0
bottomwals = 0
globvar = 0
scorevar = 0

def set_globvar_to_one():
    global globvar
    globvar = 1

def add_score_var():
    global scorevar
    scorevar += 1

class TopWall(Sprite):
    top = ImageAsset("images/top.png")
    def __init__(self, position):
        super().__init__(TopWall.top, position)

    def step(self):
        if self.x <= -100 and self.y == -201:
            self.y = -420
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -420:
            self.y = -25
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -25:
            self.y = -290
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -290:
            self.y = -135
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -135:
            self.y = -400
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -400:
            self.y = -200
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -200:
            self.y = -420
            self.x = 400
            add_score_var()
        if self.visible == True:
            collides = self.collidingWithSprites(DeadBird)
            if len(collides):
                self.x = 30
                set_globvar_to_one()
            else:
                self.x -= 3
        if globvar == 1:
            self.x = 30

class BottomWall(Sprite):
    bottom = ImageAsset("images/bottom.png")
    def __init__(self, position):
        super().__init__(BottomWall.bottom, position)
        self.iswalls = True

    def step(self):
        if self.x <= -100 and self.y == 424:
            self.y = 205
            self.x = 400
        elif self.x <= -100 and self.y == 205:
            self.y = 600
            self.x = 400
        elif self.x <= -100 and self.y == 600:
            self.y = 335
            self.x = 400
        elif self.x <= -100 and self.y == 335:
            self.y = 490
            self.x = 400
        elif self.x <= -100 and self.y == 490:
            self.y = 225
            self.x = 400
        elif self.x <= -100 and self.y == 225:
            self.y = 425
            self.x = 400
        elif self.x <= -100 and self.y == 425:
            self.y = 205
            self.x = 400
        if self.visible == True:
            collides = self.collidingWithSprites(DeadBird)
            if len(collides):
                self.x = 30
                set_globvar_to_one()
            else:
                self.x -= 3
        if globvar == 1:
            self.x = 30

class Bird(Sprite):
    asset1 = ImageAsset("images/0.png")
    asset2 = ImageAsset("images/1.png")
    asset3 = ImageAsset("images/2.png")

    def __init__(self, position):
        super().__init__(Bird.asset1, position)
        self.birdy = 350
        self.gravity = .1
        self.wallx = 400
        self.wally = random.randint(-420, 0)
        self.bottomwally = self.wally + 625
        CrappyApp.listenKeyEvent("keydown", "space", self.Jump)

    def step(self):
        self.gravity += .15
        self.birdy += self.gravity
        self.y = self.birdy
        self.spawnwalls()
        self.wallx -= 3
        if self.visible == True:
            collides = self.collidingWithSprites(TopWall)
            collides1 = self.collidingWithSprites(BottomWall)
            if len(collides):
                self.die()
            if len(collides1):
                self.die()
        if self.visible == True:
            if self.y <= 10 or self.y >= 665:
                self.die()

    def Jump(self, event):
        self.birdy -= 50
        self.gravity = .1

    def spawnwalls(self):
        if self.wallx == 400:
            TopWall((self.wallx, -201))
            BottomWall((self.wallx, 424))

    def die(self):
        self.visible = False
        DeadBird((self.x, self.y))
        self.destroy()
        print(scorevar)
        if scorevar < 10:
            print("you fucking nigger can't play this shit. Seriously, go to therapy you autistic fuck.")

class DeadBird(Sprite):

    asset = ImageAsset("images/dead.png")

    def __init__(self, position):
        super().__init__(DeadBird.asset, position)

class CrappyApp(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        bg_asset = ImageAsset("images/background.png")
        bg = Sprite(bg_asset, (0,0))
        Bird((20, 350))

    def step(self):
        for bird in self.getSpritesbyClass(Bird):
            bird.step()
        for birds in self.getSpritesbyClass(TopWall):
            birds.step()
        for bottoms in self.getSpritesbyClass(BottomWall):
            bottoms.step()

myapp = CrappyApp(400, 708)
myapp.run()<commit_msg>Updated from Brython Server: 1/6/2016 2:00:10 PM<commit_after>"""
Name: Dimitri Somoff
Final Project: Flappy bird remake "Crappy Bird"
"""
from ggame import App, Color, LineStyle, Sprite, ImageAsset, Frame, SoundAsset
import random

wals = 0
bottomwals = 0
globvar = 0
scorevar = 0

def set_globvar_to_one():
    global globvar
    globvar = 1

def add_score_var():
    global scorevar
    scorevar += 1

class TopWall(Sprite):
    top = ImageAsset("images/top.png")
    def __init__(self, position):
        super().__init__(TopWall.top, position)

    def step(self):
        if self.x <= -100 and self.y == -201:
            self.y = -420
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -420:
            self.y = -25
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -25:
            self.y = -290
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -290:
            self.y = -135
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -135:
            self.y = -400
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -400:
            self.y = -200
            self.x = 400
            add_score_var()
        elif self.x <= -100 and self.y == -200:
            self.y = -420
            self.x = 400
            add_score_var()
        if self.visible == True:
            collides = self.collidingWithSprites(DeadBird)
            if len(collides):
                self.x = 30
                set_globvar_to_one()
            else:
                self.x -= 3
        if globvar == 1:
            self.x = 30

class BottomWall(Sprite):
    bottom = ImageAsset("images/bottom.png")
    def __init__(self, position):
        super().__init__(BottomWall.bottom, position)
        self.iswalls = True

    def step(self):
        if self.x <= -100 and self.y == 424:
            self.y = 205
            self.x = 400
        elif self.x <= -100 and self.y == 205:
            self.y = 600
            self.x = 400
        elif self.x <= -100 and self.y == 600:
            self.y = 335
            self.x = 400
        elif self.x <= -100 and self.y == 335:
            self.y = 490
            self.x = 400
        elif self.x <= -100 and self.y == 490:
            self.y = 225
            self.x = 400
        elif self.x <= -100 and self.y == 225:
            self.y = 425
            self.x = 400
        elif self.x <= -100 and self.y == 425:
            self.y = 205
            self.x = 400
        if self.visible == True:
            collides = self.collidingWithSprites(DeadBird)
            if len(collides):
                self.x = 30
                set_globvar_to_one()
            else:
                self.x -= 3
        if globvar == 1:
            self.x = 30

class Bird(Sprite):
    asset1 = ImageAsset("images/0.png")
    asset2 = ImageAsset("images/1.png")
    asset3 = ImageAsset("images/2.png")

    def __init__(self, position):
        super().__init__(Bird.asset1, position)
        self.birdy = 350
        self.gravity = .1
        self.wallx = 400
        self.wally = random.randint(-420, 0)
        self.bottomwally = self.wally + 625
        CrappyApp.listenKeyEvent("keydown", "space", self.Jump)

    def step(self):
        self.gravity += .15
        self.birdy += self.gravity
        self.y = self.birdy
        self.spawnwalls()
        self.wallx -= 3
        if self.visible == True:
            collides = self.collidingWithSprites(TopWall)
            collides1 = self.collidingWithSprites(BottomWall)
            if len(collides):
                self.die()
            if len(collides1):
                self.die()
        if self.visible == True:
            if self.y <= 10 or self.y >= 665:
                self.die()

    def Jump(self, event):
        self.birdy -= 50
        self.gravity = .1

    def spawnwalls(self):
        if self.wallx == 400:
            TopWall((self.wallx, -201))
            BottomWall((self.wallx, 424))

    def die(self):
        self.visible = False
        DeadBird((self.x, self.y))
        self.destroy()
        print(scorevar)
        if scorevar < 1:
            print("you fucking nigger can't play this shit. Seriously, go to therapy you autistic fuck.")

class DeadBird(Sprite):

    asset = ImageAsset("images/dead.png")

    def __init__(self, position):
        super().__init__(DeadBird.asset, position)

class CrappyApp(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        bg_asset = ImageAsset("images/background.png")
        bg = Sprite(bg_asset, (0,0))
        Bird((20, 350))

    def step(self):
        for bird in self.getSpritesbyClass(Bird):
            bird.step()
        for birds in self.getSpritesbyClass(TopWall):
            birds.step()
        for bottoms in self.getSpritesbyClass(BottomWall):
            bottoms.step()

myapp = CrappyApp(400, 708)
myapp.run()<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 18-10-2017 16:11
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad for the chapter 3 from the book
"""

import pandas
import numpy as np
import math
from sklearn.linear_model import LogisticRegression

# Helper functions
# Categorical-to-numerical function from chapter 2 changed to automatically add column names
def cat_to_num(data):
    categories = np.unique(data)
    features = {}
    for cat in categories:
        binary = (data == cat)
        features["{}_{}".format(data.name, cat)] = binary.astype("int")
    return pandas.DataFrame(features)

def prepare_data(data):
    """
    Takes a dataframe of raw data and returns ML model features
    :param data: dataframe of raw data
    :return: ML model features
    """
    # Initially, we build a model only no the available numerical values
    features = data.drop(["PassengerId", "Survived", "Fare", "Name", "Sex", "Ticket", "Cabin", "Embarked"], axis=1)
    # Setting missing age values to -1
    features["Age"] = data["Age"].fillna(-1)
    # Adding the sqrt of the fare feature
    features["sqrt_Fare"] = math.sqrt(data["Fare"])
    # Adding gender categorical value
    features = features.join(cat_to_num(data["Sex"]))
    # Adding Embarked categorical value
    features = features.join(cat_to_num(data["Embarked"]))
    # ML model features are now ready
    return features

# Read the Titanic sample data
sample_data_titanic = pandas.read_csv("../book_code/data/titanic.csv")
print("---> Sample data - Titanic, #{} entries".format(len(sample_data_titanic)))
print("... Sample ...\n"
      "{}\n"
      "... END of Sample ...".format(sample_data_titanic[:5]))

# We make a 80/20% train/test split of the data
data_train = sample_data_titanic[:int(0.8 * len(sample_data_titanic))]
data_test = sample_data_titanic[int(0.8 * len(sample_data_titanic)):]

# ML training model
ml_training_model = prepare_data(data_train)
print("---> ML Training model\n"
      "... SAMPLE ...\n"
      "{}\n"
      "... END of SAMPLE".format(ml_training_model[:5]))
model = LogisticRegression()
model.fit(ml_training_model, data_train['Survived'])
<commit_msg>print features drop list<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 18-10-2017 16:11
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad for the chapter 3 from the book
"""

import pandas
import numpy as np
import math
from sklearn.linear_model import LogisticRegression

# Helper functions
# Categorical-to-numerical function from chapter 2 changed to automatically add column names
def cat_to_num(data):
    categories = np.unique(data)
    features = {}
    for cat in categories:
        binary = (data == cat)
        features["{}_{}".format(data.name, cat)] = binary.astype("int")
    return pandas.DataFrame(features)

def prepare_data(data):
    """
    Takes a dataframe of raw data and returns ML model features
    :param data: dataframe of raw data
    :return: ML model features
    """
    # Initially, we build a model only no the available numerical values
    features_drop_list = ["PassengerId", "Survived", "Fare", "Name", "Sex", "Ticket", "Cabin", "Embarked"]
    print("+++> Drop features: {}".format(features_drop_list))
    features = data.drop(features_drop_list, axis=1)
    # Setting missing age values to -1
    features["Age"] = data["Age"].fillna(-1)
    # Adding the sqrt of the fare feature
    features["sqrt_Fare"] = math.sqrt(data["Fare"])
    # Adding gender categorical value
    features = features.join(cat_to_num(data["Sex"]))
    # Adding Embarked categorical value
    features = features.join(cat_to_num(data["Embarked"]))
    # ML model features are now ready
    return features

# Read the Titanic sample data
sample_data_titanic = pandas.read_csv("../book_code/data/titanic.csv")
print("---> Sample data - Titanic, #{} entries".format(len(sample_data_titanic)))
print("... Sample ...\n"
      "{}\n"
      "... END of Sample ...".format(sample_data_titanic[:5]))

# We make a 80/20% train/test split of the data
data_train = sample_data_titanic[:int(0.8 * len(sample_data_titanic))]
data_test = sample_data_titanic[int(0.8 * len(sample_data_titanic)):]

# ML training model
ml_training_model = prepare_data(data_train)
print("---> ML Training model\n"
      "... SAMPLE ...\n"
      "{}\n"
      "... END of SAMPLE".format(ml_training_model[:5]))
model = LogisticRegression()
model.fit(ml_training_model, data_train['Survived'])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# pylint: disable=superfluous-parens
#
"""
haproxyadmin.backend
~~~~~~~~~~~~~~~~~~~~

This module provides the :class:`Backend <.Backend>` class which allows to
run operation for a backend.

"""
from .utils import (calculate, cmd_across_all_procs, compare_values)
from .server import Server

class Backend(object):
    """A container for a backend across several HAProxy processes."""
    BACKEND_METRICS = [
        'act',
        'bck',
        'bin',
        'bout',
        'chkdown',
        'cli_abrt',
        'comp_byp',
        'comp_in',
        'comp_out',
        'comp_rsp',
        'ctime',
        'downtime',
        'dreq',
        'dresp',
        'econ',
        'eresp',
        'hrsp_1xx',
        'hrsp_2xx',
        'hrsp_3xx',
        'hrsp_4xx',
        'hrsp_5xx',
        'hrsp_other',
        'lastchg',
        'lastsess',
        'lbtot',
        'qcur',
        'qmax',
        'qtime',
        'rate',
        'rate_max',
        'rtime',
        'scur',
        'slim',
        'smax',
        'srv_abrt',
        'status',
        'stot',
        'ttime',
        'weight',
        'wredis',
        'wretr',
    ]

    def __init__(self, backend_per_proc):
        self._backend_per_proc = backend_per_proc
        self._name = self._backend_per_proc[0].name

    # built-in comparison operator is adjusted
    def __eq__(self, other):
        if isinstance(other, Backend):
            return (self.name == other.name)
        elif isinstance(other, str):
            return (self.name == other)
        else:
            return False

    def __ne__(self, other):
        return (not self.__eq__(other))

    def servers(self, name=None):
        """Return Server object for each server.

        :param name: (optional) servername to look for. Defaults to None.
        :type name: string
        :return: A list of :class:`Server <Server>` objects
        :rtype: list
        """
        return_list = []

        # store _Server objects for each server as it is reported by each
        # process.
        # key: name of the server
        # value: a list of _Server object
        servers_across_hap_processes = {}

        # Get a list of servers (_Server objects) per process
        for backend in self._backend_per_proc:
            for server in backend.servers(name):
                if server.name not in servers_across_hap_processes:
                    servers_across_hap_processes[server.name] = []
                servers_across_hap_processes[server.name].append(server)

        # For each server build a Server object
        for server_per_proc in servers_across_hap_processes.values():
            return_list.append(Server(server_per_proc, self.name))

        return return_list

    def server(self, name):
        """Return a Server object

        :param name: Name of the server
        :type name: string
        :return: :class:`Server <Server>` object
        :rtype: haproxyadmin.Server
        """
        server = self.servers(name)
        if len(server) == 1:
            return server[0]
        elif len(server) == 0:
            raise ValueError("Could not find server")
        else:
            raise ValueError("Found more than one server, this is a bug!")

    def metric(self, name):
        """Return the value of a metric.

        Performs a calculation on the metric across all HAProxy processes.
        The type of calculation is either sum or avg and defined in
        utils.METRICS_SUM and utils.METRICS_AVG.

        :param name: Name of the metric, any of BACKEND_METRICS
        :type name: ``string``
        :return: Value of the metric after the appropriate calculation
          has been performed.
        :rtype: number, either ``integer`` or ``float``.
        :raise: ValueError when a given metric is not found.
        """
        metrics = []
        if name not in Backend.BACKEND_METRICS:
            raise ValueError("{} is not valid metric".format(name))

        metrics = [x.metric(name) for x in self._backend_per_proc]

        return calculate(name, metrics)

    @property
    def name(self):
        """Return the name of the backend.

        :rtype: string
        """
        return self._name

    @property
    def process_nb(self):
        """Return a list of process number in which backend is configured.

        :rtype: list
        """
        process_numbers = []
        for backend in self._backend_per_proc:
            process_numbers.append(backend.process_nb)

        return process_numbers

    @property
    def requests(self):
        """Return the number of requests.

        :rtype: integer
        """
        return self.metric('lbtot')

    def requests_per_process(self):
        """Return the number of requests for the backend per process.

        :return: a list of tuples with 2 elements

          #. process number of HAProxy
          #. requests

        :rtype: ``list``

        """
        results = cmd_across_all_procs(self._backend_per_proc, 'metric', 'lbtot')

        return results

    def stats_per_process(self):
        """Return all stats of the backend per process.

        :return: a list of tuples with 2 elements

          #. process number
          #. a dict with all stats

        :rtype: ``list``

        """
        values = cmd_across_all_procs(self._backend_per_proc, 'stats')

        return values

    @property
    def status(self):
        """Return the status of the backend.

        :rtype: ``string``
        :raise: :class:`IncosistentData` exception if status is different
          per process.

        """
        results = cmd_across_all_procs(self._backend_per_proc, 'metric', 'status')

        return compare_values(results)
<commit_msg>backend.py: remove status from BACKEND_METRICS<commit_after># -*- coding: utf-8 -*-
#
# pylint: disable=superfluous-parens
#
"""
haproxyadmin.backend
~~~~~~~~~~~~~~~~~~~~

This module provides the :class:`Backend <.Backend>` class which allows to
run operation for a backend.

"""
from .utils import (calculate, cmd_across_all_procs, compare_values)
from .server import Server

class Backend(object):
    """A container for a backend across several HAProxy processes."""
    BACKEND_METRICS = [
        'act',
        'bck',
        'bin',
        'bout',
        'chkdown',
        'cli_abrt',
        'comp_byp',
        'comp_in',
        'comp_out',
        'comp_rsp',
        'ctime',
        'downtime',
        'dreq',
        'dresp',
        'econ',
        'eresp',
        'hrsp_1xx',
        'hrsp_2xx',
        'hrsp_3xx',
        'hrsp_4xx',
        'hrsp_5xx',
        'hrsp_other',
        'lastchg',
        'lastsess',
        'lbtot',
        'qcur',
        'qmax',
        'qtime',
        'rate',
        'rate_max',
        'rtime',
        'scur',
        'slim',
        'smax',
        'srv_abrt',
        'stot',
        'ttime',
        'weight',
        'wredis',
        'wretr',
    ]

    def __init__(self, backend_per_proc):
        self._backend_per_proc = backend_per_proc
        self._name = self._backend_per_proc[0].name

    # built-in comparison operator is adjusted
    def __eq__(self, other):
        if isinstance(other, Backend):
            return (self.name == other.name)
        elif isinstance(other, str):
            return (self.name == other)
        else:
            return False

    def __ne__(self, other):
        return (not self.__eq__(other))

    def servers(self, name=None):
        """Return Server object for each server.

        :param name: (optional) servername to look for. Defaults to None.
        :type name: string
        :return: A list of :class:`Server <Server>` objects
        :rtype: list
        """
        return_list = []

        # store _Server objects for each server as it is reported by each
        # process.
        # key: name of the server
        # value: a list of _Server object
        servers_across_hap_processes = {}

        # Get a list of servers (_Server objects) per process
        for backend in self._backend_per_proc:
            for server in backend.servers(name):
                if server.name not in servers_across_hap_processes:
                    servers_across_hap_processes[server.name] = []
                servers_across_hap_processes[server.name].append(server)

        # For each server build a Server object
        for server_per_proc in servers_across_hap_processes.values():
            return_list.append(Server(server_per_proc, self.name))

        return return_list

    def server(self, name):
        """Return a Server object

        :param name: Name of the server
        :type name: string
        :return: :class:`Server <Server>` object
        :rtype: haproxyadmin.Server
        """
        server = self.servers(name)
        if len(server) == 1:
            return server[0]
        elif len(server) == 0:
            raise ValueError("Could not find server")
        else:
            raise ValueError("Found more than one server, this is a bug!")

    def metric(self, name):
        """Return the value of a metric.

        Performs a calculation on the metric across all HAProxy processes.
        The type of calculation is either sum or avg and defined in
        utils.METRICS_SUM and utils.METRICS_AVG.

        :param name: Name of the metric, any of BACKEND_METRICS
        :type name: ``string``
        :return: Value of the metric after the appropriate calculation
          has been performed.
        :rtype: number, either ``integer`` or ``float``.
        :raise: ValueError when a given metric is not found.
        """
        metrics = []
        if name not in Backend.BACKEND_METRICS:
            raise ValueError("{} is not valid metric".format(name))

        metrics = [x.metric(name) for x in self._backend_per_proc]

        return calculate(name, metrics)

    @property
    def name(self):
        """Return the name of the backend.

        :rtype: string
        """
        return self._name

    @property
    def process_nb(self):
        """Return a list of process number in which backend is configured.

        :rtype: list
        """
        process_numbers = []
        for backend in self._backend_per_proc:
            process_numbers.append(backend.process_nb)

        return process_numbers

    @property
    def requests(self):
        """Return the number of requests.

        :rtype: integer
        """
        return self.metric('lbtot')

    def requests_per_process(self):
        """Return the number of requests for the backend per process.

        :return: a list of tuples with 2 elements

          #. process number of HAProxy
          #. requests

        :rtype: ``list``

        """
        results = cmd_across_all_procs(self._backend_per_proc, 'metric', 'lbtot')

        return results

    def stats_per_process(self):
        """Return all stats of the backend per process.

        :return: a list of tuples with 2 elements

          #. process number
          #. a dict with all stats

        :rtype: ``list``

        """
        values = cmd_across_all_procs(self._backend_per_proc, 'stats')

        return values

    @property
    def status(self):
        """Return the status of the backend.

        :rtype: ``string``
        :raise: :class:`IncosistentData` exception if status is different
          per process.

        """
        results = cmd_across_all_procs(self._backend_per_proc, 'metric', 'status')

        return compare_values(results)
<|endoftext|>