<commit_before>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#
class CommonVariables:
    utils_path_name = 'Utils'
    extension_name = 'AzureDiskEncryptionForLinuxTest'
    extension_version = '0.1.0.999139'
    extension_type = extension_name
    extension_media_link = 'https://amextpaas.blob.core.windows.net/prod/' + extension_name + '-' + str(extension_version) + '.zip'
    extension_label = 'Windows Azure VMEncryption Extension for Linux IaaS'
    extension_description = extension_label

    """
    disk/file system related
    """
    sector_size = 512
    luks_header_size = 4096 * 512
    default_block_size = 52428800
    min_filesystem_size_support = 52428800 * 3
    #TODO for the sles 11, we should use the ext3
    default_file_system = 'ext4'
    default_mount_name = 'encrypted_disk'
    dev_mapper_root = '/dev/mapper/'
    disk_by_id_root = '/dev/disk/by-id'
    BekVolumeFileSystem = 'vfat'

    """
    parameter key names
    """
    PassphraseFileNameKey = 'BekFileName'
    KeyEncryptionKeyURLKey = 'KeyEncryptionKeyURL'
    KeyVaultURLKey = 'KeyVaultURL'
    AADClientIDKey = 'AADClientID'
    KeyEncryptionAlgorithmKey = 'KeyEncryptionAlgorithm'
    DiskFormatQuerykey = "DiskFormatQuery"
    PassphraseKey = 'Passphrase'
    BekVolumeFileSystemKey = "BekVolumeFileSystem"

    """
    value for VolumeType could be OS or Data
    """
    VolumeTypeKey = 'VolumeType'
    AADClientSecretKey = 'AADClientSecret'
    SecretUriKey = 'SecretUri'

    VolumeTypeOS = 'OS'
    VolumeTypeData = 'Data'
    VolumeTypeAll = 'All'

    SupportedVolumeTypes = [ VolumeTypeOS, VolumeTypeData, VolumeTypeAll ]

    """
    command types
    """
    EnableEncryption = 'EnableEncryption'
    EnableEncryptionFormat = 'EnableEncryptionFormat'
    DisableEncryption = 'DisableEncryption'

    """
    encryption config keys
    """
    EncryptionEncryptionOperationKey = 'EncryptionOperation'
    EncryptionDecryptionOperationKey = 'DecryptionOperation'
    EncryptionVolumeTypeKey = 'VolumeType'
    EncryptionDiskFormatQueryKey = 'DiskFormatQuery'

    """
    crypt ongoing item config keys
    """
    OngoingItemMapperNameKey = 'MapperName'
    OngoingItemHeaderFilePathKey = 'HeaderFilePath'
    OngoingItemOriginalDevNamePathKey = 'DevNamePath'
    OngoingItemOriginalDevPathKey = 'DevicePath'
    OngoingItemPhaseKey = 'Phase'
    OngoingItemHeaderSliceFilePathKey = 'HeaderSliceFilePath'
    OngoingItemFileSystemKey = 'FileSystem'
    OngoingItemMountPointKey = 'MountPoint'
    OngoingItemDeviceSizeKey = 'Size'
    OngoingItemCurrentSliceIndexKey = 'CurrentSliceIndex'
    OngoingItemFromEndKey = 'FromEnd'
    OngoingItemCurrentDestinationKey = 'CurrentDestination'
    OngoingItemCurrentTotalCopySizeKey = 'CurrentTotalCopySize'
    OngoingItemCurrentLuksHeaderFilePathKey = 'CurrentLuksHeaderFilePath'
    OngoingItemCurrentSourcePathKey = 'CurrentSourcePath'
    OngoingItemCurrentBlockSizeKey = 'CurrentBlockSize'

    """
    encryption phase devinitions
    """
    EncryptionPhaseBackupHeader = 'BackupHeader'
    EncryptionPhaseCopyData = 'EncryptingData'
    EncryptionPhaseRecoverHeader = 'RecoverHeader'
    EncryptionPhaseEncryptDevice = 'EncryptDevice'
    EncryptionPhaseDone = 'Done'

    """
    decryption phase constants
    """
    DecryptionPhaseCopyData = 'DecryptingData'
    DecryptionPhaseDone = 'Done'

    """
    logs related
    """
    InfoLevel = 'Info'
    WarningLevel = 'Warning'
    ErrorLevel = 'Error'

    """
    error codes
    """
    extension_success_status = 'success'
    extension_error_status = 'error'
    process_success = 0
    success = 0
    os_not_supported = 1
    luks_format_error = 2
    scsi_number_not_found = 3
    device_not_blank = 4
    environment_error = 5
    luks_open_error = 6
    mkfs_error = 7
    folder_conflict_error = 8
    mount_error = 9
    mount_point_not_exists = 10
    passphrase_too_long_or_none = 11
    parameter_error = 12
    create_encryption_secret_failed = 13
    encrypttion_already_enabled = 14
    passphrase_file_not_found = 15
    command_not_support = 16
    volue_type_not_support = 17
    copy_data_error = 18
    encryption_failed = 19
    tmpfs_error = 20
    backup_slice_file_error = 21
    unmount_oldroot_error = 22
    unknown_error = 100

class TestHooks:
    search_not_only_ide = False
    use_hard_code_passphrase = False
    hard_code_passphrase = "Quattro!"

class DeviceItem(object):
    def __init__(self):
        #NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL
        self.name = None
        self.type = None
        self.file_system = None
        self.mount_point = None
        self.label = None
        self.uuid = None
        self.model = None
        self.size = None
    def __str__(self):
        return "name:" + str(self.name) + " type:" + str(self.type) + " fstype:" + str(self.file_system) + " mountpoint:" + str(self.mount_point) + " label:" + str(self.label) + " model:" + str(self.model)

class CryptItem(object):
    def __init__(self):
        self.mapper_name = None
        self.dev_path = None
        self.mount_point = None
        self.file_system = None
        self.luks_header_path = None
        self.uses_cleartext_key = None

    def __str__(self):
        return ("name: " + str(self.mapper_name) + " dev_path:" + str(self.dev_path) +
                " mount_point:" + str(self.mount_point) + " file_system:" + str(self.file_system) +
                " luks_header_path:" + str(self.luks_header_path) +
                " uses_cleartext_key:" + str(self.uses_cleartext_key))
<commit_msg>Bump version number<commit_after>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#
class CommonVariables:
    utils_path_name = 'Utils'
    extension_name = 'AzureDiskEncryptionForLinuxTest'
    extension_version = '0.1.0.999140'
    extension_type = extension_name
    extension_media_link = 'https://amextpaas.blob.core.windows.net/prod/' + extension_name + '-' + str(extension_version) + '.zip'
    extension_label = 'Windows Azure VMEncryption Extension for Linux IaaS'
    extension_description = extension_label

    """
    disk/file system related
    """
    sector_size = 512
    luks_header_size = 4096 * 512
    default_block_size = 52428800
    min_filesystem_size_support = 52428800 * 3
    #TODO for the sles 11, we should use the ext3
    default_file_system = 'ext4'
    default_mount_name = 'encrypted_disk'
    dev_mapper_root = '/dev/mapper/'
    disk_by_id_root = '/dev/disk/by-id'
    BekVolumeFileSystem = 'vfat'

    """
    parameter key names
    """
    PassphraseFileNameKey = 'BekFileName'
    KeyEncryptionKeyURLKey = 'KeyEncryptionKeyURL'
    KeyVaultURLKey = 'KeyVaultURL'
    AADClientIDKey = 'AADClientID'
    KeyEncryptionAlgorithmKey = 'KeyEncryptionAlgorithm'
    DiskFormatQuerykey = "DiskFormatQuery"
    PassphraseKey = 'Passphrase'
    BekVolumeFileSystemKey = "BekVolumeFileSystem"

    """
    value for VolumeType could be OS or Data
    """
    VolumeTypeKey = 'VolumeType'
    AADClientSecretKey = 'AADClientSecret'
    SecretUriKey = 'SecretUri'

    VolumeTypeOS = 'OS'
    VolumeTypeData = 'Data'
    VolumeTypeAll = 'All'

    SupportedVolumeTypes = [ VolumeTypeOS, VolumeTypeData, VolumeTypeAll ]

    """
    command types
    """
    EnableEncryption = 'EnableEncryption'
    EnableEncryptionFormat = 'EnableEncryptionFormat'
    DisableEncryption = 'DisableEncryption'

    """
    encryption config keys
    """
    EncryptionEncryptionOperationKey = 'EncryptionOperation'
    EncryptionDecryptionOperationKey = 'DecryptionOperation'
    EncryptionVolumeTypeKey = 'VolumeType'
    EncryptionDiskFormatQueryKey = 'DiskFormatQuery'

    """
    crypt ongoing item config keys
    """
    OngoingItemMapperNameKey = 'MapperName'
    OngoingItemHeaderFilePathKey = 'HeaderFilePath'
    OngoingItemOriginalDevNamePathKey = 'DevNamePath'
    OngoingItemOriginalDevPathKey = 'DevicePath'
    OngoingItemPhaseKey = 'Phase'
    OngoingItemHeaderSliceFilePathKey = 'HeaderSliceFilePath'
    OngoingItemFileSystemKey = 'FileSystem'
    OngoingItemMountPointKey = 'MountPoint'
    OngoingItemDeviceSizeKey = 'Size'
    OngoingItemCurrentSliceIndexKey = 'CurrentSliceIndex'
    OngoingItemFromEndKey = 'FromEnd'
    OngoingItemCurrentDestinationKey = 'CurrentDestination'
    OngoingItemCurrentTotalCopySizeKey = 'CurrentTotalCopySize'
    OngoingItemCurrentLuksHeaderFilePathKey = 'CurrentLuksHeaderFilePath'
    OngoingItemCurrentSourcePathKey = 'CurrentSourcePath'
    OngoingItemCurrentBlockSizeKey = 'CurrentBlockSize'

    """
    encryption phase devinitions
    """
    EncryptionPhaseBackupHeader = 'BackupHeader'
    EncryptionPhaseCopyData = 'EncryptingData'
    EncryptionPhaseRecoverHeader = 'RecoverHeader'
    EncryptionPhaseEncryptDevice = 'EncryptDevice'
    EncryptionPhaseDone = 'Done'

    """
    decryption phase constants
    """
    DecryptionPhaseCopyData = 'DecryptingData'
    DecryptionPhaseDone = 'Done'

    """
    logs related
    """
    InfoLevel = 'Info'
    WarningLevel = 'Warning'
    ErrorLevel = 'Error'

    """
    error codes
    """
    extension_success_status = 'success'
    extension_error_status = 'error'
    process_success = 0
    success = 0
    os_not_supported = 1
    luks_format_error = 2
    scsi_number_not_found = 3
    device_not_blank = 4
    environment_error = 5
    luks_open_error = 6
    mkfs_error = 7
    folder_conflict_error = 8
    mount_error = 9
    mount_point_not_exists = 10
    passphrase_too_long_or_none = 11
    parameter_error = 12
    create_encryption_secret_failed = 13
    encrypttion_already_enabled = 14
    passphrase_file_not_found = 15
    command_not_support = 16
    volue_type_not_support = 17
    copy_data_error = 18
    encryption_failed = 19
    tmpfs_error = 20
    backup_slice_file_error = 21
    unmount_oldroot_error = 22
    unknown_error = 100

class TestHooks:
    search_not_only_ide = False
    use_hard_code_passphrase = False
    hard_code_passphrase = "Quattro!"

class DeviceItem(object):
    def __init__(self):
        #NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL
        self.name = None
        self.type = None
        self.file_system = None
        self.mount_point = None
        self.label = None
        self.uuid = None
        self.model = None
        self.size = None
    def __str__(self):
        return "name:" + str(self.name) + " type:" + str(self.type) + " fstype:" + str(self.file_system) + " mountpoint:" + str(self.mount_point) + " label:" + str(self.label) + " model:" + str(self.model)

class CryptItem(object):
    def __init__(self):
        self.mapper_name = None
        self.dev_path = None
        self.mount_point = None
        self.file_system = None
        self.luks_header_path = None
        self.uses_cleartext_key = None

    def __str__(self):
        return ("name: " + str(self.mapper_name) + " dev_path:" + str(self.dev_path) +
                " mount_point:" + str(self.mount_point) + " file_system:" + str(self.file_system) +
                " luks_header_path:" + str(self.luks_header_path) +
                " uses_cleartext_key:" + str(self.uses_cleartext_key))
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import sys
import Adafruit_DHT

# Parse command line parameters.
sensor_args = { '11': Adafruit_DHT.DHT11,
                '22': Adafruit_DHT.DHT22,
                '2302': Adafruit_DHT.AM2302 }

if len(sys.argv) >= 2:
        pin = sys.argv[1]
else:
        sys.exit(1)

filename = os.path.join('/dev/shm', 'dht-' + pin)
try:
  with open(filename) as f:
    h, t = [float(x) for x in next(f).split()]
except:
  h = None
  t = None

def get():
  humidity, temperature = Adafruit_DHT.read_retry(sensor_args['22'], pin, retries=11)
  if humidity is not None and temperature is not None and 0 <= humidity <= 100.0:
    return (humidity, temperature)
  return (None, None)

if h is None:
  # Need to seed the stored values
  h, t = get()

for i in (1,2,3,4):
  h2, t2 = get()
  if h is not None and 0 <= h <= 100.0 and h2 is not None and 0 <= h2 <= 100.0 and (abs(t - t2) < 1 or time.time() - os.stat(filename).st_mtime > 3600):
    # Good readings
    with open(filename, "w+") as f:
      print('{0:0.1f} {1:0.1f}'.format(h2, t2), file=f)
    print('{0:0.1f} {1:0.1f}'.format(h2, t2))
    sys.exit(0)
  else:
    print('Attempt {0:d} failed'.format(i), file=sys.stderr)

print('U U')
sys.exit(1)
<commit_msg>import time<commit_after>#!/usr/bin/python3

import os
import sys
import time
import Adafruit_DHT

# Parse command line parameters.
sensor_args = { '11': Adafruit_DHT.DHT11,
                '22': Adafruit_DHT.DHT22,
                '2302': Adafruit_DHT.AM2302 }

if len(sys.argv) >= 2:
        pin = sys.argv[1]
else:
        sys.exit(1)

filename = os.path.join('/dev/shm', 'dht-' + pin)
try:
  with open(filename) as f:
    h, t = [float(x) for x in next(f).split()]
except:
  h = None
  t = None

def get():
  humidity, temperature = Adafruit_DHT.read_retry(sensor_args['22'], pin, retries=11)
  if humidity is not None and temperature is not None and 0 <= humidity <= 100.0:
    return (humidity, temperature)
  return (None, None)

if h is None:
  # Need to seed the stored values
  h, t = get()

for i in (1,2,3,4):
  h2, t2 = get()
  if h is not None and 0 <= h <= 100.0 and h2 is not None and 0 <= h2 <= 100.0 and (abs(t - t2) < 1 or time.time() - os.stat(filename).st_mtime > 3600):
    # Good readings
    with open(filename, "w+") as f:
      print('{0:0.1f} {1:0.1f}'.format(h2, t2), file=f)
    print('{0:0.1f} {1:0.1f}'.format(h2, t2))
    sys.exit(0)
  else:
    print('Attempt {0:d} failed'.format(i), file=sys.stderr)

print('U U')
sys.exit(1)
<|endoftext|>


<commit_before># Compare auto threshold methods for a grayscale image

import os
import cv2
import numpy as np
from plantcv.plantcv import params
from plantcv.plantcv.transform import resize_factor
from plantcv.plantcv import plot_image
from plantcv.plantcv import print_image
from plantcv.plantcv import fatal_error
from plantcv.plantcv.threshold import mean
from plantcv.plantcv.threshold import otsu
from plantcv.plantcv.threshold import gaussian
from plantcv.plantcv.threshold import triangle

def auto_threshold_methods(gray_img, grid_img=True, object_type="light"):
    """ Compare auto threshold methods for a grayscale image

    Inputs:
    gray_img     = Grayscale image data
    grid_img     = Whether or not to compile masks into a single plot
    object_type  = "light" or "dark" (default: "light")
                   - If object is lighter than the background then standard thresholding is done
                   - If object is darker than the background then inverse thresholding is done

    Returns:
    labeled_imgs = List of labeled plotting images

    :param gray_img: numpy.ndarray
    :param grid_img: bool
    :param object_type: str
    :return labeled_imgs: list

    """
    # Check that the image is grayscale
    if not len(np.shape(gray_img)) == 2:
        fatal_error("Input image is not grayscale!")

    # Store and disable debug mode
    debug = params.debug
    params.debug = None

    # Initialize threshold method names, mask list, final images
    method_names = ["Gaussian", "Mean", "Otsu", "Triangle"]
    all_methods = []
    labeled_imgs = []

    # Set starting location for labeling different masks
    y = int(np.shape(gray_img)[0] / 8)
    x = int(np.shape(gray_img)[1] / 8)

    # Create mask imgs from each thresholding method
    all_methods.append(gaussian(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(mean(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(otsu(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(triangle(gray_img=gray_img, max_value=255, object_type=object_type, xstep=1))

    # Plot labels of each colorspace on the corresponding img
    for i, method in enumerate(all_methods):
        converted_img = cv2.cvtColor(method, cv2.COLOR_GRAY2RGB)
        labeled = cv2.putText(img=converted_img, text=method_names[i], org=(x, y),
                              fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                              fontScale=params.text_size, color=(255, 0, 255), thickness=params.text_thickness)
        # Reset debug mode
        params.debug = debug
        if params.debug == "print":
            # If debug is print, save the image to a file
            print_image(labeled, os.path.join(params.debug_outdir, str(params.device) + "_" +
                                              method_names[i] + "_vis_thresholds.png"))
        elif params.debug == "plot":
            # If debug is plot, print to the plotting device
            plot_image(labeled)
        labeled_imgs.append(labeled)

    if grid_img:
        # Store and disable debug mode
        debug = params.debug
        params.debug = None
        # Compile images together into one
        top_row = np.hstack([labeled_imgs[0], labeled_imgs[1]])
        bot_row = np.hstack([labeled_imgs[2], labeled_imgs[3]])
        plotting_img = np.vstack([top_row, bot_row])
        labeled_imgs.append(plotting_img)
        plotting_img = resize_factor(plotting_img, factors=(0.5, 0.5))
        # Reset debug mode
        params.debug = debug
        if params.debug == "print":
            # If debug is print, save the image to a file
            print_image(plotting_img, os.path.join(params.debug_outdir, str(params.device) + "_vis_all_thresholds.png"))
        elif params.debug == "plot":
            # If debug is plot, print to the plotting device
            plot_image(plotting_img)

    return labeled_imgs
<commit_msg>Update auto_threshold_methods.py<commit_after># Compare auto threshold methods for a grayscale image

import os
import cv2
import numpy as np
from plantcv.plantcv import params
from plantcv.plantcv.transform import resize_factor
from plantcv.plantcv import plot_image
from plantcv.plantcv import print_image
from plantcv.plantcv import fatal_error
from plantcv.plantcv._debug import _debug
from plantcv.plantcv.threshold import mean
from plantcv.plantcv.threshold import otsu
from plantcv.plantcv.threshold import gaussian
from plantcv.plantcv.threshold import triangle

def auto_threshold_methods(gray_img, grid_img=True, object_type="light"):
    """ Compare auto threshold methods for a grayscale image

    Inputs:
    gray_img     = Grayscale image data
    grid_img     = Whether or not to compile masks into a single plot
    object_type  = "light" or "dark" (default: "light")
                   - If object is lighter than the background then standard thresholding is done
                   - If object is darker than the background then inverse thresholding is done

    Returns:
    labeled_imgs = List of labeled plotting images

    :param gray_img: numpy.ndarray
    :param grid_img: bool
    :param object_type: str
    :return labeled_imgs: list

    """
    # Check that the image is grayscale
    if not len(np.shape(gray_img)) == 2:
        fatal_error("Input image is not grayscale!")

    # Store and disable debug mode
    debug = params.debug
    params.debug = None

    # Initialize threshold method names, mask list, final images
    method_names = ["Gaussian", "Mean", "Otsu", "Triangle"]
    all_methods = []
    labeled_imgs = []

    # Set starting location for labeling different masks
    y = int(np.shape(gray_img)[0] / 8)
    x = int(np.shape(gray_img)[1] / 8)

    # Create mask imgs from each thresholding method
    all_methods.append(gaussian(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(mean(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(otsu(gray_img=gray_img, max_value=255, object_type=object_type))
    all_methods.append(triangle(gray_img=gray_img, max_value=255, object_type=object_type, xstep=1))

    # Plot labels of each colorspace on the corresponding img
    for i, method in enumerate(all_methods):
        converted_img = cv2.cvtColor(method, cv2.COLOR_GRAY2RGB)
        labeled = cv2.putText(img=converted_img, text=method_names[i], org=(x, y),
                              fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                              fontScale=params.text_size, color=(255, 0, 255), thickness=params.text_thickness)
        # Reset debug mode
        params.debug = debug
        _debug(visual=labeled, filename=os.path.join(params.debug_outdir, str(params.device) + "_" +
                                              method_names[i] + "_vis_thresholds.png"))
        labeled_imgs.append(labeled)

    if grid_img:
        # Store and disable debug mode
        debug = params.debug
        params.debug = None
        # Compile images together into one
        top_row = np.hstack([labeled_imgs[0], labeled_imgs[1]])
        bot_row = np.hstack([labeled_imgs[2], labeled_imgs[3]])
        plotting_img = np.vstack([top_row, bot_row])
        labeled_imgs.append(plotting_img)
        plotting_img = resize_factor(plotting_img, factors=(0.5, 0.5))
        # Reset debug mode
        params.debug = debug
        _debug(visual=plotting_img, filename=os.path.join(params.debug_outdir, str(params.device) +
                                                          "_vis_all_thresholds.png"))

    return labeled_imgs
<|endoftext|>


<commit_before>"""

NOTE: for now this is just intended to be run as python -m tests from the
compiler directory (parent directory of this file).
"""
import logging
import os
import select
import subprocess
import unittest

class CompilerTestBase(unittest.TestCase):
    """Contains the code that actually runs tests and reports their results.
    """

    # TODO: make this a command line argument

    @classmethod
    def setUpClass(cls):

        # Static Configuration
        # TODO: Move static configuration to a config file.
        verbose = False
        self.compiler_name = "cradle"
        self.paths_to_test = ["./", "../"]

        self.compiler_path = cls.get_compiler_path(compiler_name)

        # Command that is used to run the compiler.
        self.run_compiler_string = "{}".format(
            compiler_path,)

        # This is the command and arguments that will build the compiler
        self.file_extension = "pas"
        build_compiler_commands = ["fpc", "{}.{}".format(
            compiler_path,
            self.file_extension)]

        if cls is CompilerTestBase:
            cls._build_compiler()

    @staticmethod
    def potential_paths(potential_paths):
        for path in potential_paths:
            yield "{}{}".format(path, self.compiler_name)

    @classmethod
    def get_compiler_path(cls, compiler_name):
        """

        # TODO: change into a class attribute function.
        """
        # Currently only supporting calling from test directory and project
        # directory.
        for compiler_path in cls.potential_paths(paths_to_test):
            if os.path.isfile(compiler_path):
                return compiler_path

        raise RuntimeError("No compiler could be found to test. Either the"
            "compiler paths are incorrectly configured or the compiler name"
            "is incorrectly configured.")

    @classmethod
    def _build_compiler(cls):
        """Runs the compliation step before running any tests. If this fails,
        abort the tests.

        Subprocess handling code from
        https://stackoverflow.com/questions/18273962/python-subprocess-call-hangs

        # TODO: COMPILE USING THE PROJECT MAKEFILE!!!
        # No, really, that needs to be the next workflow improvement.
        """
        logger = logging.getLogger(__name__)
        is_running = lambda: compile_process.poll() is None

        compile_process = subprocess.Popen(
            cls.build_compiler_commands,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,)

        # Grab all the output from stdout and stderr and log it
        while is_running():
            rlist, wlist, xlist = select.select(
                [compile_process.stdout, compile_process.stderr], [], [], 1)

        # Log stdout, but don't spam the log
        if compile_process.stdout in rlist and cls.verbose:
            # Adjust the number of bytes read however you like, 1024 seems to work
            # pretty well for me.
            logger.debug(compile_process.stdout.read(1024))

        # Log stderr, always
        if compile_process.stderr in rlist:
            # Same as with stdout, adjust the bytes read as needed.
            logger.error(compile_process.stderr.read(1024))

    def clean_formatting(self, raw_text):
        """Simple string cleaner that ignores all prior whitespace and left
        hand side whitespace.
        """
        raw_text = raw_text.decode("utf-8")
        # Remove all whitespace leading up to actual text.
        raw_text = raw_text.strip()
        split_text = raw_text.split("\n")
        cleaned_text = [line.lstrip() for line in split_text]
        return "\n".join(cleaned_text)

    def run_test(self, test_program, expected_assembly):
        sub_process = subprocess.Popen(
            [self.run_compiler_string],
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        try:
            test_program = bytes(test_program, "utf-8")
        except AttributeError:
            # Already a string, do nothing.
            pass
        actual_assembly = sub_process.communicate(test_program)
        actual_assembly = self.clean_formatting(actual_assembly[0])
        expected_assembly = self.clean_formatting(expected_assembly)
        if self.verbose:
            # TODO: switch to logging statements.
            print("------- actual assembly ----------")
            print(actual_assembly)
            print("------- expected assembly ----------")
            print(expected_assembly)
            print("------- end ----------")
        self.assertEqual(actual_assembly, expected_assembly)

class TestInitialization(CompilerTestBase):

    def test_single_literal(self):
        test_program = "1"
        expected_assembly = b"""
        .text
        .globl _main
            _main:
            subq $8, %rsp
            movq $1, %rax
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

    def test_blank_program(self):
        test_program = ""
        expected_assembly = b"""
        .text
        .globl _main
        _main:
            subq $8, %rsp
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

class TestArithmetic(CompilerTestBase):

    division_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $2, %rax
        push %rax
        movq $2, %rax
        movq %rax, %rbx
        pop %rax
        xor %rdx, %rdx
        div %rbx
        movq $0, %rdi
        call _exit
    """

    multiplication_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        mul %rbx
        movq $0, %rdi
        call _exit
    """

    addition_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        add %rax, %rbx
        movq $0, %rdi
        call _exit
    """

    subtraction_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        sub %rax, %rbx
        movq $0, %rdi
        call _exit
    """
    def test_addition_no_space(self):
        test_program = "1+2"
        self.run_test(
            test_program,
            expected_assembly=self.addition_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_addition_spaces(self):
        test_program = "1 + 2"
        self.run_test(
            test_program,
            expected_assembly=self.addition_assembly)

    @unittest.skip("Errors not included yet.")
    def test_addition_missing_left_literal(self):
        test_program = "+2"
        expected_assembly = b"Error: Addop Expected"
        self.run_test(test_program, expected_assembly)

    @unittest.skip("Errors not included yet.")
    def test_addition_missing_right_literal(self):
        test_program = "1+"
        expected_assembly = b"Error: Addop Expected"
        self.run_test(test_program, expected_assembly)

    def test_subtraction_no_space(self):
        test_program = "1-2"
        self.run_test(
            test_program,
            expected_assembly=self.subtraction_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_subtraction_spaces(self):
        test_program = "1 - 2"
        self.run_test(
            test_program,
            expected_assembly=self.subtraction_assembly)

    @unittest.skip("Errors not included yet.")
    def test_subtraction_missing_right_literal(self):
        test_program = "1-"
        expected_assembly = b"Error: Subop Expected"
        self.run_test(test_program, expected_assembly)

    def test_multiplciation_no_spaces(self):
        test_program = "1*2"
        self.run_test(
            test_program,
            expected_assembly=self.multiplication_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_multiplciation_spaces(self):
        test_program = "1 * 2"
        self.run_test(
            test_program,
            expected_assembly=self.multiplication_assembly)

    @unittest.skip("Errors not included yet.")
    def test_invalid_mul_operation(self):
        test_program = "1*"
        expected_assembly = b"Error: Mulop Expected"
        self.run_test(test_program, expected_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_division_spaces(self):
        test_program = "2 / 2"
        self.run_test(
            test_program,
            expected_assembly=self.division_assembly)

    def test_division_no_spaces(self):
        test_program = "2/2"
        self.run_test(
            test_program,
            expected_assembly=self.division_assembly)

    @unittest.skip("Errors not included yet.")
    def test_invalid_div_operation(self):
        test_program = "1/"
        expected_assembly = b"Error: Divop Expected"
        self.run_test(test_program, expected_assembly)

class TestParenthese(CompilerTestBase):

    pre_boiler_plate = b"""
    """
    post_boiler_plate = b"""
    """
    addition_assembly = b"""
    """

    def constant_assembly(integer):
        return b""

    def test_parens_no_expression(self):
        test_program = "()"
        expected_assembly = b"""
        .text
        .globl _main
        _main:
            subq $8, %rsp
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_signle_digit(self):
        test_program = "(1)"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_sigle_expression(self):
        test_program = "(1+2)"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_nested_expression(self):
        test_program = "(1(2+(3)))"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_broad_expressions(self):
        test_program = "(1+(2+3))+4"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

if __name__ == "__main__":
    logging.basicConfig()
    unittest.main()
<commit_msg>Finished refactor of tests to support moving to separate files.<commit_after>"""

NOTE: for now this is just intended to be run as python -m tests from the
compiler directory (parent directory of this file).
"""
import logging
import os
import select
import subprocess
import unittest

class CompilerTestBase(unittest.TestCase):
    """Contains the code that actually runs tests and reports their results.
    """

    # TODO: make this a command line argument

    @classmethod
    def setUpClass(cls):

        # Static Configuration
        # TODO: Move static configuration to a config file.
        cls.verbose = False
        cls.compiler_name = "cradle"
        cls.paths_to_test = ["./", "../"]
        # End static configuration

        cls.compiler_path = cls.get_compiler_path(cls.compiler_name)

        # Command that is used to run the compiler.
        cls.run_compiler_string = "{}".format(cls.compiler_path)

        # This is the command and arguments that will build the compiler
        cls.file_extension = "pas"
        build_compiler_commands = ["fpc", "{}.{}".format(
            cls.compiler_path,
            cls.file_extension)]

        if cls is CompilerTestBase:
            cls._build_compiler()

    def potential_paths(cls, potential_paths):
        for path in potential_paths:
            yield "{}{}".format(path, cls.compiler_name)

    @classmethod
    def get_compiler_path(cls, compiler_name):
        """

        # TODO: change into a class attribute function.
        """
        # Currently only supporting calling from test directory and project
        # directory.
        for compiler_path in cls.potential_paths(cls, cls.paths_to_test):
            if os.path.isfile(compiler_path):
                return compiler_path

        raise RuntimeError("No compiler could be found to test. Either the"
            "compiler paths are incorrectly configured or the compiler name"
            "is incorrectly configured.")

    @classmethod
    def _build_compiler(cls):
        """Runs the compliation step before running any tests. If this fails,
        abort the tests.

        Subprocess handling code from
        https://stackoverflow.com/questions/18273962/python-subprocess-call-hangs

        # TODO: COMPILE USING THE PROJECT MAKEFILE!!!
        # No, really, that needs to be the next workflow improvement.
        """
        logger = logging.getLogger(__name__)
        is_running = lambda: compile_process.poll() is None

        compile_process = subprocess.Popen(
            cls.build_compiler_commands,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,)

        # Grab all the output from stdout and stderr and log it
        while is_running():
            rlist, wlist, xlist = select.select(
                [compile_process.stdout, compile_process.stderr], [], [], 1)

        # Log stdout, but don't spam the log
        if compile_process.stdout in rlist and cls.verbose:
            # Adjust the number of bytes read however you like, 1024 seems to work
            # pretty well for me.
            logger.debug(compile_process.stdout.read(1024))

        # Log stderr, always
        if compile_process.stderr in rlist:
            # Same as with stdout, adjust the bytes read as needed.
            logger.error(compile_process.stderr.read(1024))

    def clean_formatting(self, raw_text):
        """Simple string cleaner that ignores all prior whitespace and left
        hand side whitespace.
        """
        raw_text = raw_text.decode("utf-8")
        # Remove all whitespace leading up to actual text.
        raw_text = raw_text.strip()
        split_text = raw_text.split("\n")
        cleaned_text = [line.lstrip() for line in split_text]
        return "\n".join(cleaned_text)

    def run_test(self, test_program, expected_assembly):
        sub_process = subprocess.Popen(
            [self.run_compiler_string],
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        try:
            test_program = bytes(test_program, "utf-8")
        except AttributeError:
            # Already a string, do nothing.
            pass
        actual_assembly = sub_process.communicate(test_program)
        actual_assembly = self.clean_formatting(actual_assembly[0])
        expected_assembly = self.clean_formatting(expected_assembly)
        if self.verbose:
            # TODO: switch to logging statements.
            print("------- actual assembly ----------")
            print(actual_assembly)
            print("------- expected assembly ----------")
            print(expected_assembly)
            print("------- end ----------")
        self.assertEqual(actual_assembly, expected_assembly)

class TestInitialization(CompilerTestBase):

    def test_single_literal(self):
        test_program = "1"
        expected_assembly = b"""
        .text
        .globl _main
            _main:
            subq $8, %rsp
            movq $1, %rax
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

    def test_blank_program(self):
        test_program = ""
        expected_assembly = b"""
        .text
        .globl _main
        _main:
            subq $8, %rsp
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

class TestArithmetic(CompilerTestBase):

    division_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $2, %rax
        push %rax
        movq $2, %rax
        movq %rax, %rbx
        pop %rax
        xor %rdx, %rdx
        div %rbx
        movq $0, %rdi
        call _exit
    """

    multiplication_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        mul %rbx
        movq $0, %rdi
        call _exit
    """

    addition_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        add %rax, %rbx
        movq $0, %rdi
        call _exit
    """

    subtraction_assembly = b"""
    .text
    .globl _main
    _main:
        subq $8, %rsp
        movq $1, %rax
        push %rax
        movq $2, %rax
        pop %rbx
        sub %rax, %rbx
        movq $0, %rdi
        call _exit
    """
    def test_addition_no_space(self):
        test_program = "1+2"
        self.run_test(
            test_program,
            expected_assembly=self.addition_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_addition_spaces(self):
        test_program = "1 + 2"
        self.run_test(
            test_program,
            expected_assembly=self.addition_assembly)

    @unittest.skip("Errors not included yet.")
    def test_addition_missing_left_literal(self):
        test_program = "+2"
        expected_assembly = b"Error: Addop Expected"
        self.run_test(test_program, expected_assembly)

    @unittest.skip("Errors not included yet.")
    def test_addition_missing_right_literal(self):
        test_program = "1+"
        expected_assembly = b"Error: Addop Expected"
        self.run_test(test_program, expected_assembly)

    def test_subtraction_no_space(self):
        test_program = "1-2"
        self.run_test(
            test_program,
            expected_assembly=self.subtraction_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_subtraction_spaces(self):
        test_program = "1 - 2"
        self.run_test(
            test_program,
            expected_assembly=self.subtraction_assembly)

    @unittest.skip("Errors not included yet.")
    def test_subtraction_missing_right_literal(self):
        test_program = "1-"
        expected_assembly = b"Error: Subop Expected"
        self.run_test(test_program, expected_assembly)

    def test_multiplciation_no_spaces(self):
        test_program = "1*2"
        self.run_test(
            test_program,
            expected_assembly=self.multiplication_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_multiplciation_spaces(self):
        test_program = "1 * 2"
        self.run_test(
            test_program,
            expected_assembly=self.multiplication_assembly)

    @unittest.skip("Errors not included yet.")
    def test_invalid_mul_operation(self):
        test_program = "1*"
        expected_assembly = b"Error: Mulop Expected"
        self.run_test(test_program, expected_assembly)

    @unittest.skip("Spaces not included yet.")
    def test_division_spaces(self):
        test_program = "2 / 2"
        self.run_test(
            test_program,
            expected_assembly=self.division_assembly)

    def test_division_no_spaces(self):
        test_program = "2/2"
        self.run_test(
            test_program,
            expected_assembly=self.division_assembly)

    @unittest.skip("Errors not included yet.")
    def test_invalid_div_operation(self):
        test_program = "1/"
        expected_assembly = b"Error: Divop Expected"
        self.run_test(test_program, expected_assembly)

class TestParenthese(CompilerTestBase):

    pre_boiler_plate = b"""
    """
    post_boiler_plate = b"""
    """
    addition_assembly = b"""
    """

    def constant_assembly(integer):
        return b""

    def test_parens_no_expression(self):
        test_program = "()"
        expected_assembly = b"""
        .text
        .globl _main
        _main:
            subq $8, %rsp
            movq $0, %rdi
            call _exit
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_signle_digit(self):
        test_program = "(1)"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_sigle_expression(self):
        test_program = "(1+2)"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_nested_expression(self):
        test_program = "(1(2+(3)))"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

    def test_parens_broad_expressions(self):
        test_program = "(1+(2+3))+4"
        expected_assembly = b"""
        """
        self.run_test(test_program, expected_assembly)

if __name__ == "__main__":
    logging.basicConfig()
    unittest.main()
<|endoftext|>


<commit_before># Backport from Django 1.9a1
from psycopg2.extras import Json

from django.contrib.postgres import lookups
from django.core import exceptions
from django.db.models import Field, Transform
from django.utils.translation import ugettext_lazy as _

from keybar.utils import json

class HasKey(lookups.PostgresSimpleLookup):
    lookup_name = 'has_key'
    operator = '?'

class HasKeys(lookups.PostgresSimpleLookup):
    lookup_name = 'has_keys'
    operator = '?&'

class HasAnyKeys(lookups.PostgresSimpleLookup):
    lookup_name = 'has_any_keys'
    operator = '?|'

class JSONField(Field):
    empty_strings_allowed = False
    description = _('A JSON object')
    default_error_messages = {
        'invalid': _("Value must be valid JSON."),
    }

    def db_type(self, connection):
        return 'jsonb'

    def get_transform(self, name):
        transform = super(JSONField, self).get_transform(name)
        if transform:
            return transform
        return KeyTransformFactory(name)

    def get_prep_value(self, value):
        if value is not None:
            return Json(value)
        return value

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in ('has_key', 'has_keys', 'has_any_keys'):
            return value
        if isinstance(value, (dict, list)):
            return Json(value)
        return super(JSONField, self).get_prep_lookup(lookup_type, value)

    def validate(self, value, model_instance):
        super(JSONField, self).validate(value, model_instance)
        try:
            json.dumps(value)
        except TypeError:
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def value_to_string(self, obj):
        value = self.value_from_object(obj)
        return value

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.JSONField}
        defaults.update(kwargs)
        return super(JSONField, self).formfield(**defaults)

JSONField.register_lookup(lookups.DataContains)
JSONField.register_lookup(lookups.ContainedBy)
JSONField.register_lookup(HasKey)
JSONField.register_lookup(HasKeys)
JSONField.register_lookup(HasAnyKeys)

class KeyTransform(Transform):

    def __init__(self, key_name, *args, **kwargs):
        super(KeyTransform, self).__init__(*args, **kwargs)
        self.key_name = key_name

    def as_sql(self, compiler, connection):
        key_transforms = [self.key_name]
        previous = self.lhs
        while isinstance(previous, KeyTransform):
            key_transforms.insert(0, previous.key_name)
            previous = previous.lhs
        lhs, params = compiler.compile(previous)
        if len(key_transforms) > 1:
            return "{} #> %s".format(lhs), [key_transforms] + params
        try:
            int(self.key_name)
        except ValueError:
            lookup = "'%s'" % self.key_name
        else:
            lookup = "%s" % self.key_name
        return "%s -> %s" % (lhs, lookup), params

class KeyTransformFactory(object):

    def __init__(self, key_name):
        self.key_name = key_name

    def __call__(self, *args, **kwargs):
        return KeyTransform(self.key_name, *args, **kwargs)
<commit_msg>Smaller cleanups<commit_after># Backport from Django 1.9a1
from django.contrib.postgres import lookups
from django.core import exceptions
from django.db.models import Field, Transform
from django.utils.translation import ugettext_lazy as _
from psycopg2.extras import Json

from keybar.utils import json

class HasKey(lookups.PostgresSimpleLookup):
    lookup_name = 'has_key'
    operator = '?'

class HasKeys(lookups.PostgresSimpleLookup):
    lookup_name = 'has_keys'
    operator = '?&'

class HasAnyKeys(lookups.PostgresSimpleLookup):
    lookup_name = 'has_any_keys'
    operator = '?|'

class JSONField(Field):
    empty_strings_allowed = False
    description = _('A JSON object')
    default_error_messages = {
        'invalid': _("Value must be valid JSON."),
    }

    def db_type(self, connection):
        return 'jsonb'

    def get_transform(self, name):
        transform = super(JSONField, self).get_transform(name)
        if transform:
            return transform
        return KeyTransformFactory(name)

    def get_prep_value(self, value):
        if value is not None:
            return Json(value)
        return value

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type in ('has_key', 'has_keys', 'has_any_keys'):
            return value
        if isinstance(value, (dict, list)):
            return Json(value)
        return super(JSONField, self).get_prep_lookup(lookup_type, value)

    def validate(self, value, model_instance):
        super(JSONField, self).validate(value, model_instance)
        try:
            json.dumps(value)
        except TypeError:
            raise exceptions.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'value': value},
            )

    def value_to_string(self, obj):
        value = self.value_from_object(obj)
        return value

    def formfield(self, **kwargs):
        # TODO: Use ´postgres.forms.JSONField´ if needed.
        defaults = {'form_class': forms.CharField}
        defaults.update(kwargs)
        return super(JSONField, self).formfield(**defaults)

JSONField.register_lookup(lookups.DataContains)
JSONField.register_lookup(lookups.ContainedBy)
JSONField.register_lookup(HasKey)
JSONField.register_lookup(HasKeys)
JSONField.register_lookup(HasAnyKeys)

class KeyTransform(Transform):

    def __init__(self, key_name, *args, **kwargs):
        super(KeyTransform, self).__init__(*args, **kwargs)
        self.key_name = key_name

    def as_sql(self, compiler, connection):
        key_transforms = [self.key_name]
        previous = self.lhs
        while isinstance(previous, KeyTransform):
            key_transforms.insert(0, previous.key_name)
            previous = previous.lhs
        lhs, params = compiler.compile(previous)
        if len(key_transforms) > 1:
            return "{} #> %s".format(lhs), [key_transforms] + params
        try:
            int(self.key_name)
        except ValueError:
            lookup = "'%s'" % self.key_name
        else:
            lookup = "%s" % self.key_name
        return "%s -> %s" % (lhs, lookup), params

class KeyTransformFactory(object):

    def __init__(self, key_name):
        self.key_name = key_name

    def __call__(self, *args, **kwargs):
        return KeyTransform(self.key_name, *args, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env jasy

import shutil, json

dist = "build"

# Configure fields
session.setField("es5", True)
session.permutateField("debug")

formatting = Formatting('semicolon', 'comma')

@task
def clean():
    logging.info("Clearing cache...")
    session.clearCache()

@task
def distclean():
    logging.info("Clearing cache...")
    session.clearCache()

    if os.path.exists(dist):
        logging.info("Deleting build folder...")
        shutil.rmtree(dist)

@task
def build(dist="build"):

    # Write kernel script
    resolver = Resolver()
    resolver.addClassName("api.Browser")
    assets = Asset(resolver.getIncludedClasses()).exportBuild()
    includedByKernel = storeKernel("%s/script/kernel.js" % dist, assets=assets, debug=False)

    # Copy files from source
    updateFile("source/index.html", "%s/index.html" % dist)

    # Rewrite template as jsonp
    for tmpl in ["main", "error", "entry", "type", "params", "info", "origin", "tags"]:
        jsonTemplate = json.dumps({ "template" : open("source/tmpl/%s.mustache" % tmpl).read() })
        writeFile("%s/tmpl/%s.js" % (dist, tmpl), "apibrowser.callback(%s, '%s.mustache')" % (jsonTemplate, tmpl))

    # Process every possible permutation
    for permutation in session.permutate():

        # Resolving dependencies
        resolver = Resolver()
        resolver.addClassName("api.Browser")
        resolver.excludeClasses(includedByKernel)

        # Compressing classes
        classes = Sorter(resolver).getSortedClasses()
        compressedCode = storeCompressed("%s/script/browser-%s.js" % (dist, getPermutation().getChecksum()), classes, bootCode="apibrowser=new api.Browser();")

    # Write API data
    writer = ApiWriter().write("%s/data" % dist, compact=False, callback="apibrowser.callback")

@task
def source():
    dist = "source"

    # Prepare assets
    resolver = Resolver()
    resolver.addClassName("api.Browser")
    assets = Asset(resolver.getIncludedClasses()).exportSource()

    # Write kernel script
    includedByKernel = storeKernel("%s/script/kernel.js" % dist, assets=assets, formatting=formatting, debug=True)

    # Rewrite template as jsonp
    for tmpl in ["main", "error", "entry", "type", "params", "info", "origin", "tags"]:
        jsonTemplate = json.dumps({ "template" : open("source/tmpl/%s.mustache" % tmpl).read() })
        writeFile("%s/tmpl/%s.js" % (dist, tmpl), "apibrowser.callback(%s, '%s.mustache')" % (jsonTemplate, tmpl))

    # Process every possible permutation
    for permutation in session.permutate():

        # Resolving dependencies
        resolver = Resolver(permutation)
        resolver.addClassName("api.Browser")
        resolver.excludeClasses(includedByKernel)

        # Compressing classes
        classes = Sorter(resolver, permutation).getSortedClasses()
        compressedCode = storeSourceLoader("%s/script/browser-%s.js" % (dist, permutation.getChecksum()), classes, bootCode="apibrowser=new api.Browser();")

    # Write API data
    writer = ApiWriter().write("%s/data" % dist, compact=False, callback="apibrowser.callback")

<commit_msg>Cleanups for new jasy<commit_after>#!/usr/bin/env jasy

import shutil, json

dist = "build"

# Configure fields
session.setField("es5", True)
session.permutateField("debug")

formatting = Formatting('semicolon', 'comma')

@task
def clean():
    logging.info("Clearing cache...")
    session.clearCache()

@task
def distclean():
    logging.info("Clearing cache...")
    session.clearCache()

    if os.path.exists(dist):
        logging.info("Deleting build folder...")
        shutil.rmtree(dist)

@task
def build(dist="build"):

    # Write kernel script
    resolver = Resolver().addClassName("api.Browser")
    assets = Asset(resolver.getIncludedClasses()).exportBuild()
    includedByKernel = storeKernel("%s/script/kernel.js" % dist, assets=assets, debug=False)

    # Copy files from source
    updateFile("source/index.html", "%s/index.html" % dist)

    # Rewrite template as jsonp
    for tmpl in ["main", "error", "entry", "type", "params", "info", "origin", "tags"]:
        jsonTemplate = json.dumps({ "template" : open("source/tmpl/%s.mustache" % tmpl).read() })
        writeFile("%s/tmpl/%s.js" % (dist, tmpl), "apibrowser.callback(%s, '%s.mustache')" % (jsonTemplate, tmpl))

    # Process every possible permutation
    for permutation in session.permutate():

        # Resolving dependencies
        resolver = Resolver()
        resolver.addClassName("api.Browser")
        resolver.excludeClasses(includedByKernel)

        # Compressing classes
        classes = Sorter(resolver).getSortedClasses()
        compressedCode = storeCompressed("%s/script/browser-%s.js" % (dist, getPermutation().getChecksum()), classes, bootCode="apibrowser=new api.Browser();")

    # Write API data
    writer = ApiWriter().write("%s/data" % dist, compact=False, callback="apibrowser.callback")

@task
def source():
    dist = "source"

    # Write kernel script
    resolver = Resolver().addClassName("api.Browser")
    assets = Asset(resolver.getIncludedClasses()).exportSource()
    includedByKernel = storeKernel("%s/script/kernel.js" % dist, assets=assets, debug=True)

    # Rewrite template as jsonp
    for tmpl in ["main", "error", "entry", "type", "params", "info", "origin", "tags"]:
        jsonTemplate = json.dumps({ "template" : open("source/tmpl/%s.mustache" % tmpl).read() })
        writeFile("%s/tmpl/%s.js" % (dist, tmpl), "apibrowser.callback(%s, '%s.mustache')" % (jsonTemplate, tmpl))

    # Process every possible permutation
    for permutation in session.permutate():

        # Resolving dependencies
        resolver = Resolver()
        resolver.addClassName("api.Browser")
        resolver.excludeClasses(includedByKernel)

        # Compressing classes
        classes = Sorter(resolver).getSortedClasses()
        compressedCode = storeSourceLoader("%s/script/browser-%s.js" % (dist, getPermutation().getChecksum()), classes, bootCode="apibrowser=new api.Browser();")

    # Write API data
    writer = ApiWriter().write("%s/data" % dist, compact=False, callback="apibrowser.callback")

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Watcher base exception handling.

Includes decorator for re-raising Watcher-type exceptions.

SHOULD include dedicated exception logging.

"""

import functools
import sys

from keystoneclient import exceptions as keystone_exceptions
from oslo_config import cfg
from oslo_log import log as logging
import six

from watcher._i18n import _, _LE

LOG = logging.getLogger(__name__)

exc_log_opts = [
    cfg.BoolOpt('fatal_exception_format_errors',
                default=False,
                help='Make exception message format errors fatal.'),
]

CONF = cfg.CONF
CONF.register_opts(exc_log_opts)

def wrap_keystone_exception(func):
    """Wrap keystone exceptions and throw Watcher specific exceptions."""
    @functools.wraps(func)
    def wrapped(*args, **kw):
        try:
            return func(*args, **kw)
        except keystone_exceptions.AuthorizationFailure:
            raise AuthorizationFailure(
                client=func.__name__, reason=sys.exc_info()[1])
        except keystone_exceptions.ClientException:
            raise AuthorizationFailure(
                client=func.__name__,
                reason=(_('Unexpected keystone client error occurred: %s')
                        % sys.exc_info()[1]))
    return wrapped

class WatcherException(Exception):
    """Base Watcher Exception

    To correctly use this class, inherit from it and define
    a 'msg_fmt' property. That msg_fmt will get printf'd
    with the keyword arguments provided to the constructor.

    """
    msg_fmt = _("An unknown exception occurred")
    code = 500
    headers = {}
    safe = False

    def __init__(self, message=None, **kwargs):
        self.kwargs = kwargs

        if 'code' not in self.kwargs:
            try:
                self.kwargs['code'] = self.code
            except AttributeError:
                pass

        if not message:
            try:
                message = self.msg_fmt % kwargs
            except Exception as e:
                # kwargs doesn't match a variable in msg_fmt
                # log the issue and the kwargs
                LOG.exception(_LE('Exception in string format operation'))
                for name, value in kwargs.items():
                    LOG.error(_LE("%(name)s: %(value)s"),
                              {'name': name, 'value': value})

                if CONF.fatal_exception_format_errors:
                    raise e
                else:
                    # at least get the core msg_fmt out if something happened
                    message = self.msg_fmt

        super(WatcherException, self).__init__(message)

    def __str__(self):
        """Encode to utf-8 then wsme api can consume it as well"""
        if not six.PY3:
            return unicode(self.args[0]).encode('utf-8')
        else:
            return self.args[0]

    def __unicode__(self):
        return unicode(self.args[0])

    def format_message(self):
        if self.__class__.__name__.endswith('_Remote'):
            return self.args[0]
        else:
            return six.text_type(self)

class NotAuthorized(WatcherException):
    msg_fmt = _("Not authorized")
    code = 403

class PolicyNotAuthorized(NotAuthorized):
    msg_fmt = _("Policy doesn't allow %(action)s to be performed.")

class OperationNotPermitted(NotAuthorized):
    msg_fmt = _("Operation not permitted")

class Invalid(WatcherException):
    msg_fmt = _("Unacceptable parameters")
    code = 400

class ObjectNotFound(WatcherException):
    msg_fmt = _("The %(name)s %(id)s could not be found")

class Conflict(WatcherException):
    msg_fmt = _('Conflict')
    code = 409

class ResourceNotFound(ObjectNotFound):
    msg_fmt = _("The %(name)s resource %(id)s could not be found")
    code = 404

class InvalidIdentity(Invalid):
    msg_fmt = _("Expected a uuid or int but received %(identity)s")

class InvalidOperator(Invalid):
    msg_fmt = _("Filter operator is not valid: %(operator)s not "
                "in %(valid_operators)s")

class InvalidGoal(Invalid):
    msg_fmt = _("Goal %(goal)s is invalid")

class InvalidStrategy(Invalid):
    msg_fmt = _("Strategy %(strategy)s is invalid")

class InvalidUUID(Invalid):
    msg_fmt = _("Expected a uuid but received %(uuid)s")

class InvalidName(Invalid):
    msg_fmt = _("Expected a logical name but received %(name)s")

class InvalidUuidOrName(Invalid):
    msg_fmt = _("Expected a logical name or uuid but received %(name)s")

class GoalNotFound(ResourceNotFound):
    msg_fmt = _("Goal %(goal)s could not be found")

class GoalAlreadyExists(Conflict):
    msg_fmt = _("A goal with UUID %(uuid)s already exists")

class StrategyNotFound(ResourceNotFound):
    msg_fmt = _("Strategy %(strategy)s could not be found")

class StrategyAlreadyExists(Conflict):
    msg_fmt = _("A strategy with UUID %(uuid)s already exists")

class AuditTemplateNotFound(ResourceNotFound):
    msg_fmt = _("AuditTemplate %(audit_template)s could not be found")

class AuditTemplateAlreadyExists(Conflict):
    msg_fmt = _("An audit_template with UUID or name %(audit_template)s "
                "already exists")

class AuditTemplateReferenced(Invalid):
    msg_fmt = _("AuditTemplate %(audit_template)s is referenced by one or "
                "multiple audit")

class AuditTypeNotFound(Invalid):
    msg_fmt = _("Audit type %(audit_type)s could not be found")

class AuditNotFound(ResourceNotFound):
    msg_fmt = _("Audit %(audit)s could not be found")

class AuditAlreadyExists(Conflict):
    msg_fmt = _("An audit with UUID %(uuid)s already exists")

class AuditIntervalNotSpecified(Invalid):
    msg_fmt = _("Interval of audit must be specified for %(audit_type)s.")

class AuditIntervalNotAllowed(Invalid):
    msg_fmt = _("Interval of audit must not be set for %(audit_type)s.")

class AuditReferenced(Invalid):
    msg_fmt = _("Audit %(audit)s is referenced by one or multiple action "
                "plans")

class ActionPlanNotFound(ResourceNotFound):
    msg_fmt = _("ActionPlan %(action_plan)s could not be found")

class ActionPlanAlreadyExists(Conflict):
    msg_fmt = _("An action plan with UUID %(uuid)s already exists")

class ActionPlanReferenced(Invalid):
    msg_fmt = _("Action Plan %(action_plan)s is referenced by one or "
                "multiple actions")

class ActionNotFound(ResourceNotFound):
    msg_fmt = _("Action %(action)s could not be found")

class ActionAlreadyExists(Conflict):
    msg_fmt = _("An action with UUID %(uuid)s already exists")

class ActionReferenced(Invalid):
    msg_fmt = _("Action plan %(action_plan)s is referenced by one or "
                "multiple goals")

class ActionFilterCombinationProhibited(Invalid):
    msg_fmt = _("Filtering actions on both audit and action-plan is "
                "prohibited")

class EfficacyIndicatorNotFound(ResourceNotFound):
    msg_fmt = _("Efficacy indicator %(efficacy_indicator)s could not be found")

class EfficacyIndicatorAlreadyExists(Conflict):
    msg_fmt = _("An action with UUID %(uuid)s already exists")

class ScoringEngineAlreadyExists(Conflict):
    msg_fmt = _("A scoring engine with UUID %(uuid)s already exists")

class ScoringEngineNotFound(ResourceNotFound):
    msg_fmt = _("ScoringEngine %(scoring_engine)s could not be found")

class HTTPNotFound(ResourceNotFound):
    pass

class PatchError(Invalid):
    msg_fmt = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s")

# decision engine

class WorkflowExecutionException(WatcherException):
    msg_fmt = _('Workflow execution error: %(error)s')

class IllegalArgumentException(WatcherException):
    msg_fmt = _('Illegal argument')

class NoSuchMetric(WatcherException):
    msg_fmt = _('No such metric')

class NoDataFound(WatcherException):
    msg_fmt = _('No rows were returned')

class AuthorizationFailure(WatcherException):
    msg_fmt = _('%(client)s connection failed. Reason: %(reason)s')

class KeystoneFailure(WatcherException):
    msg_fmt = _("'Keystone API endpoint is missing''")

class ClusterEmpty(WatcherException):
    msg_fmt = _("The list of compute node(s) in the cluster is empty")

class MetricCollectorNotDefined(WatcherException):
    msg_fmt = _("The metrics resource collector is not defined")

class ClusterDataModelCollectionError(WatcherException):
    msg_fmt = _("The cluster data model '%(cdm)s' could not be built")

class ClusterStateNotDefined(WatcherException):
    msg_fmt = _("The cluster state is not defined")

class NoAvailableStrategyForGoal(WatcherException):
    msg_fmt = _("No strategy could be found to achieve the '%(goal)s' goal.")

class InvalidIndicatorValue(WatcherException):
    msg_fmt = _("The indicator '%(name)s' with value '%(value)s' "
                "and spec type '%(spec_type)s' is invalid.")

class GlobalEfficacyComputationError(WatcherException):
    msg_fmt = _("Could not compute the global efficacy for the '%(goal)s' "
                "goal using the '%(strategy)s' strategy.")

class NoMetricValuesForInstance(WatcherException):
    msg_fmt = _("No values returned by %(resource_id)s for %(metric_name)s.")

class NoSuchMetricForHost(WatcherException):
    msg_fmt = _("No %(metric)s metric for %(host)s found.")

# Model

class InstanceNotFound(WatcherException):
    msg_fmt = _("The instance '%(name)s' could not be found")

class ComputeNodeNotFound(WatcherException):
    msg_fmt = _("The compute node %s could not be found")

class LoadingError(WatcherException):
    msg_fmt = _("Error loading plugin '%(name)s'")

class ReservedWord(WatcherException):
    msg_fmt = _("The identifier '%(name)s' is a reserved word")

class NotSoftDeletedStateError(WatcherException):
    msg_fmt = _("The %(name)s resource %(id)s is not soft deleted")

class NegativeLimitError(WatcherException):
    msg_fmt = _("Limit should be positive")
<commit_msg>Correct watcher reraising of exception<commit_after># -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Watcher base exception handling.

Includes decorator for re-raising Watcher-type exceptions.

SHOULD include dedicated exception logging.

"""

import functools
import sys

from keystoneclient import exceptions as keystone_exceptions
from oslo_config import cfg
from oslo_log import log as logging
import six

from watcher._i18n import _, _LE

LOG = logging.getLogger(__name__)

exc_log_opts = [
    cfg.BoolOpt('fatal_exception_format_errors',
                default=False,
                help='Make exception message format errors fatal.'),
]

CONF = cfg.CONF
CONF.register_opts(exc_log_opts)

def wrap_keystone_exception(func):
    """Wrap keystone exceptions and throw Watcher specific exceptions."""
    @functools.wraps(func)
    def wrapped(*args, **kw):
        try:
            return func(*args, **kw)
        except keystone_exceptions.AuthorizationFailure:
            raise AuthorizationFailure(
                client=func.__name__, reason=sys.exc_info()[1])
        except keystone_exceptions.ClientException:
            raise AuthorizationFailure(
                client=func.__name__,
                reason=(_('Unexpected keystone client error occurred: %s')
                        % sys.exc_info()[1]))
    return wrapped

class WatcherException(Exception):
    """Base Watcher Exception

    To correctly use this class, inherit from it and define
    a 'msg_fmt' property. That msg_fmt will get printf'd
    with the keyword arguments provided to the constructor.

    """
    msg_fmt = _("An unknown exception occurred")
    code = 500
    headers = {}
    safe = False

    def __init__(self, message=None, **kwargs):
        self.kwargs = kwargs

        if 'code' not in self.kwargs:
            try:
                self.kwargs['code'] = self.code
            except AttributeError:
                pass

        if not message:
            try:
                message = self.msg_fmt % kwargs
            except Exception:
                # kwargs doesn't match a variable in msg_fmt
                # log the issue and the kwargs
                LOG.exception(_LE('Exception in string format operation'))
                for name, value in kwargs.items():
                    LOG.error(_LE("%(name)s: %(value)s"),
                              {'name': name, 'value': value})

                if CONF.fatal_exception_format_errors:
                    raise
                else:
                    # at least get the core msg_fmt out if something happened
                    message = self.msg_fmt

        super(WatcherException, self).__init__(message)

    def __str__(self):
        """Encode to utf-8 then wsme api can consume it as well"""
        if not six.PY3:
            return unicode(self.args[0]).encode('utf-8')
        else:
            return self.args[0]

    def __unicode__(self):
        return unicode(self.args[0])

    def format_message(self):
        if self.__class__.__name__.endswith('_Remote'):
            return self.args[0]
        else:
            return six.text_type(self)

class NotAuthorized(WatcherException):
    msg_fmt = _("Not authorized")
    code = 403

class PolicyNotAuthorized(NotAuthorized):
    msg_fmt = _("Policy doesn't allow %(action)s to be performed.")

class OperationNotPermitted(NotAuthorized):
    msg_fmt = _("Operation not permitted")

class Invalid(WatcherException):
    msg_fmt = _("Unacceptable parameters")
    code = 400

class ObjectNotFound(WatcherException):
    msg_fmt = _("The %(name)s %(id)s could not be found")

class Conflict(WatcherException):
    msg_fmt = _('Conflict')
    code = 409

class ResourceNotFound(ObjectNotFound):
    msg_fmt = _("The %(name)s resource %(id)s could not be found")
    code = 404

class InvalidIdentity(Invalid):
    msg_fmt = _("Expected a uuid or int but received %(identity)s")

class InvalidOperator(Invalid):
    msg_fmt = _("Filter operator is not valid: %(operator)s not "
                "in %(valid_operators)s")

class InvalidGoal(Invalid):
    msg_fmt = _("Goal %(goal)s is invalid")

class InvalidStrategy(Invalid):
    msg_fmt = _("Strategy %(strategy)s is invalid")

class InvalidUUID(Invalid):
    msg_fmt = _("Expected a uuid but received %(uuid)s")

class InvalidName(Invalid):
    msg_fmt = _("Expected a logical name but received %(name)s")

class InvalidUuidOrName(Invalid):
    msg_fmt = _("Expected a logical name or uuid but received %(name)s")

class GoalNotFound(ResourceNotFound):
    msg_fmt = _("Goal %(goal)s could not be found")

class GoalAlreadyExists(Conflict):
    msg_fmt = _("A goal with UUID %(uuid)s already exists")

class StrategyNotFound(ResourceNotFound):
    msg_fmt = _("Strategy %(strategy)s could not be found")

class StrategyAlreadyExists(Conflict):
    msg_fmt = _("A strategy with UUID %(uuid)s already exists")

class AuditTemplateNotFound(ResourceNotFound):
    msg_fmt = _("AuditTemplate %(audit_template)s could not be found")

class AuditTemplateAlreadyExists(Conflict):
    msg_fmt = _("An audit_template with UUID or name %(audit_template)s "
                "already exists")

class AuditTemplateReferenced(Invalid):
    msg_fmt = _("AuditTemplate %(audit_template)s is referenced by one or "
                "multiple audit")

class AuditTypeNotFound(Invalid):
    msg_fmt = _("Audit type %(audit_type)s could not be found")

class AuditNotFound(ResourceNotFound):
    msg_fmt = _("Audit %(audit)s could not be found")

class AuditAlreadyExists(Conflict):
    msg_fmt = _("An audit with UUID %(uuid)s already exists")

class AuditIntervalNotSpecified(Invalid):
    msg_fmt = _("Interval of audit must be specified for %(audit_type)s.")

class AuditIntervalNotAllowed(Invalid):
    msg_fmt = _("Interval of audit must not be set for %(audit_type)s.")

class AuditReferenced(Invalid):
    msg_fmt = _("Audit %(audit)s is referenced by one or multiple action "
                "plans")

class ActionPlanNotFound(ResourceNotFound):
    msg_fmt = _("ActionPlan %(action_plan)s could not be found")

class ActionPlanAlreadyExists(Conflict):
    msg_fmt = _("An action plan with UUID %(uuid)s already exists")

class ActionPlanReferenced(Invalid):
    msg_fmt = _("Action Plan %(action_plan)s is referenced by one or "
                "multiple actions")

class ActionNotFound(ResourceNotFound):
    msg_fmt = _("Action %(action)s could not be found")

class ActionAlreadyExists(Conflict):
    msg_fmt = _("An action with UUID %(uuid)s already exists")

class ActionReferenced(Invalid):
    msg_fmt = _("Action plan %(action_plan)s is referenced by one or "
                "multiple goals")

class ActionFilterCombinationProhibited(Invalid):
    msg_fmt = _("Filtering actions on both audit and action-plan is "
                "prohibited")

class EfficacyIndicatorNotFound(ResourceNotFound):
    msg_fmt = _("Efficacy indicator %(efficacy_indicator)s could not be found")

class EfficacyIndicatorAlreadyExists(Conflict):
    msg_fmt = _("An action with UUID %(uuid)s already exists")

class ScoringEngineAlreadyExists(Conflict):
    msg_fmt = _("A scoring engine with UUID %(uuid)s already exists")

class ScoringEngineNotFound(ResourceNotFound):
    msg_fmt = _("ScoringEngine %(scoring_engine)s could not be found")

class HTTPNotFound(ResourceNotFound):
    pass

class PatchError(Invalid):
    msg_fmt = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s")

# decision engine

class WorkflowExecutionException(WatcherException):
    msg_fmt = _('Workflow execution error: %(error)s')

class IllegalArgumentException(WatcherException):
    msg_fmt = _('Illegal argument')

class NoSuchMetric(WatcherException):
    msg_fmt = _('No such metric')

class NoDataFound(WatcherException):
    msg_fmt = _('No rows were returned')

class AuthorizationFailure(WatcherException):
    msg_fmt = _('%(client)s connection failed. Reason: %(reason)s')

class KeystoneFailure(WatcherException):
    msg_fmt = _("'Keystone API endpoint is missing''")

class ClusterEmpty(WatcherException):
    msg_fmt = _("The list of compute node(s) in the cluster is empty")

class MetricCollectorNotDefined(WatcherException):
    msg_fmt = _("The metrics resource collector is not defined")

class ClusterDataModelCollectionError(WatcherException):
    msg_fmt = _("The cluster data model '%(cdm)s' could not be built")

class ClusterStateNotDefined(WatcherException):
    msg_fmt = _("The cluster state is not defined")

class NoAvailableStrategyForGoal(WatcherException):
    msg_fmt = _("No strategy could be found to achieve the '%(goal)s' goal.")

class InvalidIndicatorValue(WatcherException):
    msg_fmt = _("The indicator '%(name)s' with value '%(value)s' "
                "and spec type '%(spec_type)s' is invalid.")

class GlobalEfficacyComputationError(WatcherException):
    msg_fmt = _("Could not compute the global efficacy for the '%(goal)s' "
                "goal using the '%(strategy)s' strategy.")

class NoMetricValuesForInstance(WatcherException):
    msg_fmt = _("No values returned by %(resource_id)s for %(metric_name)s.")

class NoSuchMetricForHost(WatcherException):
    msg_fmt = _("No %(metric)s metric for %(host)s found.")

# Model

class InstanceNotFound(WatcherException):
    msg_fmt = _("The instance '%(name)s' could not be found")

class ComputeNodeNotFound(WatcherException):
    msg_fmt = _("The compute node %s could not be found")

class LoadingError(WatcherException):
    msg_fmt = _("Error loading plugin '%(name)s'")

class ReservedWord(WatcherException):
    msg_fmt = _("The identifier '%(name)s' is a reserved word")

class NotSoftDeletedStateError(WatcherException):
    msg_fmt = _("The %(name)s resource %(id)s is not soft deleted")

class NegativeLimitError(WatcherException):
    msg_fmt = _("Limit should be positive")
<|endoftext|>


<commit_before>#!/usr/bin/env python

# TODO shrink not overlap ? pixel subset ?

# TODO use spectral or gdal/orfeo for dataset manipulation and machine learning ?

import coal

def generateTrainingData(miningFilename, mineralFilename, trainingFilename):

    """
    Generate training data for mining classification by comparing a GIS mining map with the region surrounding each pixel in a mineral-classified image and writing the result to a file.
    """

    # define function to select the mining layer
    def miningLayer(miningDataset):

        return ???

    # define raster function to classify mineral regions with known mines
    def pixelNeighbors(miningDataset, mineralDataset, resultDataset):

        # TODO represent as dataset or array ?

        # for each pixel in the mining and mineral-classified datasets
        for ???:

            # get the region surrounding the mineral-classified pixel as a multiband pixel
            mineralRegion = getRegionAsMultibandPixel(mineralDataset, x, y)

            # get the mining pixel
            miningPixel = ???

            # classify the region with the mining pixel
            ???

    # apply the correlation and save the result
    coal.gis.combine(pixelNeighbors, miningLayer, miningFilename, mineralFilename, trainingFilename)

def trainMiningClassifier(miningFilename, mineralFilename):

    """
    Return a trained mining classifier given a mineral classified image and a GIS mining map.
    """

    # generate training data in a temporary file
    trainingFilename = ???
    generateTrainingData(mineralFilename, miningFilename, trainingFilename)

    # load training data
    trainingData = ???(trainingFilename)

    # initialize and train a neural network using the training data
    ???(trainingData)

    # TODO delete temporary file ?

    # return trained mining clasifier
    return classifier

def classifyRegion(mineralRegion, miningClassifier):

    """
    """

    return ???

def classifyImage(mineralFilename, miningClassifier, miningFilename):

    """
    Classify a mineral-classified image using a trained mining classifier and save the results to a file.
    """

    # open the mineral-classified image

    # allocate an MxN array for the mining-classified image
    classifiedImage = numpy.ndarray(???)

    # for each pixel in the mineral-classified image
    for ???:

        # get the surrounding region as a multiband pixel
        mineralRegion = getRegionAsMultibandPixel(mineralDataset, x, y)

        # classify it
        classifiedImage[???] = classifyRegion(mineralRegion, miningClassifier)

    # save the mining-classified image to a file

def getRegionAsMultibandPixel(image, x, y):

    """
    Return a multiband pixel representing the region in an image surrounding a given pixel.
    """

    # get the surrounding region

    # represent the region as a multiband pixel
<commit_msg>coal-19: update comments<commit_after>#!/usr/bin/env python

# TODO shrink not overlap ? pixel subset ?

# TODO use spectral or gdal/orfeo for dataset manipulation and machine learning ?

# TODO reuse saveClassifier, readClassifier ?

import coal

def generateTrainingData(miningFilename, mineralFilename, trainingFilename):

    """
    Generate training data for mining classification by comparing a GIS mining map with the region surrounding each pixel in a mineral-classified image and writing the result to a file.
    """

    # define function to select the mining layer
    def miningLayer(miningDataset):

        return ???

    # define raster function to classify mineral regions with known mines
    def pixelNeighbors(miningDataset, mineralDataset, resultDataset):

        # TODO represent as dataset or array ?

        # for each pixel in the mining and mineral-classified datasets
        for ???:

            # get the region surrounding the mineral-classified pixel as a multiband pixel
            mineralRegion = getRegionAsMultibandPixel(mineralDataset, x, y)

            # get the mining pixel
            miningPixel = ???

            # classify the region with the mining pixel
            ???

    # generate and save the training data
    coal.gis.combine(pixelNeighbors, miningLayer, miningFilename, mineralFilename, trainingFilename)

def trainMiningClassifier(miningFilename, mineralFilename):

    """
    Return a trained mining classifier given a mineral classified image and a GIS mining map.
    """

    # generate training data in a temporary file
    trainingFilename = ???
    generateTrainingData(mineralFilename, miningFilename, trainingFilename)

    # load training data
    trainingData = ???(trainingFilename)

    # initialize and train a neural network using the training data
    ???(trainingData)

    # TODO delete temporary file ?

    # return trained mining clasifier
    return classifier

def classifyRegion(mineralRegion, miningClassifier):

    """
    """

    return ???

def classifyImage(mineralFilename, miningClassifier, miningFilename):

    """
    Classify a mineral-classified image using a trained mining classifier and save the results to a file.
    """

    # open the mineral-classified image

    # allocate an MxN array for the mining-classified image
    classifiedImage = numpy.ndarray(???)

    # for each pixel in the mineral-classified image
    for ???:

        # get the surrounding region as a multiband pixel
        mineralRegion = getRegionAsMultibandPixel(mineralDataset, x, y)

        # classify it
        classifiedImage[???] = classifyRegion(mineralRegion, miningClassifier)

    # save the mining-classified image to a file

def getRegionAsMultibandPixel(image, x, y):

    """
    Return a multiband pixel representing the region in an image surrounding a given pixel.
    """

    # get the surrounding region

    # represent the region as a multiband pixel
<|endoftext|>