<commit_before>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

'''
images
'''
import os
import time

from xobj import xobj

from rbuild import errors
from rbuild import pluginapi
from rbuild.pluginapi import command

class CancelImageError(errors.RbuildError):
    """Raised when there is an error canceling an image build"""

class MissingImageError(errors.RbuildError):
    """Raised when we don"t find an image"""
    template = ("Unable to find image with id '%(image)s' on %(stage)s stage"
        " of %(project)s project")
    params = ["image", "project", "stage"]

class CancelImagesCommand(command.BaseCommand):
    help = 'Cancel image build'
    paramHelp = '<id>+'

    def runCommand(self, handle, argSet, args):
        _, imageIds = self.requireParameters(args, expected=['id'],
            appendExtra=True)

        project, branch, stage = handle.Images._getProductStage()
        for imageId in imageIds:
            try:
                int(imageId)
            except ValueError:
                raise errors.BadParameterError(
                    "Cannot parse image id '%s'" % imageId)

            image = handle.facade.rbuilder.getImages(image_id=imageId,
                project=project, branch=branch, stage=stage)

            if not image:
                handle.ui.warning(str(MissingImageError(image=imageId,
                    project=project, stage=stage)))
                continue
            image = image[0]

            try:
                handle.Images.cancel(image)
            except CancelImageError as err:
                handle.ui.warning(str(err))

class DeleteImagesCommand(command.BaseCommand):
    help = 'Delete images'
    paramHelp = '<image id>+'

    def runCommand(self, handle, argSet, args):
        _, imageIds = self.requireParameters(
            args, expected=['IMAGEID'], appendExtra=True)
        for imageId in imageIds:
            try:
                int(imageId)
                handle.Images.delete(imageId)
            except ValueError:
                handle.ui.warning("Cannot parse image id '%s'" % imageId)

class LaunchCommand(command.BaseCommand):
    help = 'Launch/Deploy an image onto a target'
    paramHelp = '<image name[=version] | image id> <target name>'
    commands = ['launch', 'deploy']
    docs = {'list': 'List available targets',
            'from-file': 'Load launch/deploy descriptor from file',
            'to-file': 'Write launch/deploy descriptor to file',
            'no-watch': 'Do not wait for job to complete',
            'no-launch': 'Do not launch an image',
            }

    def addLocalParameters(self, argDef):
        argDef['list'] = '-l', command.NO_PARAM
        argDef['from-file'] = '-f', command.ONE_PARAM
        argDef['to-file'] = '-o', command.ONE_PARAM
        argDef['no-watch'] = command.NO_PARAM
        argDef['no-launch'] = command.NO_PARAM

    def runCommand(self, handle, argSet, args):
        ui = handle.ui
        rb = handle.facade.rbuilder

        listTargets = argSet.pop('list', False)
        fromFile = argSet.pop('from-file', None)
        toFile = argSet.pop('to-file', None)
        watch = not argSet.pop('no-watch', False)
        doLaunch = not argSet.pop('no-launch', False)

        if listTargets:
            ui.write('Available targets: %s' %
                     ', '.join(t.name for t in rb.getEnabledTargets()))
            return

        if fromFile:
            handle.DescriptorConfig.readConfig(fromFile)

        command, image, target = self.requireParameters(
            args, expected=['image name or id', 'target name'])

        if command == 'deploy':
            job = handle.Images.deployImage(image, target, doLaunch)
        else:
            job = handle.Images.launchImage(image, target, doLaunch)

        if watch and doLaunch:
            handle.Images.watchJob(job)

        if toFile:
            handle.DescriptorConfig.writeConfig(toFile)

class ListImagesCommand(command.ListCommand):
    help = 'list images'
    resource = 'images'
    listFields = ('image_id', 'name', 'image_type', 'architecture', 'status',
        'status_message')
    listFieldMap = dict(
        image_type=dict(accessor=lambda i: i.image_type.name),
        )
    showFieldMap = dict(
        actions=dict(hidden=True),
        build_log=dict(accessor=lambda i: i._root.build_log.id),
        created_by=dict(accessor=lambda i: i.created_by.full_name),
        files=dict(
            accessor=lambda i: ', '.join('%s: %s' % (f.title, f.url)
                                         for f in i.files),
            ),
        jobs=dict(accessor=lambda i: i._root.jobs.id),
        project=dict(accessor=lambda i: i.project.name),
        project_branch=dict(accessor=lambda i: i.project_branch.name[0]),
        # add in fields definied in listFieldMap
        **listFieldMap
        )

    def _list(self, handle, *args, **kwargs):
        resources = super(ListImagesCommand, self)._list(
            handle, *args, **kwargs)
        if resources:
            handle.ui.write('\nLatest:')
            for latest in resources._node.latest_files:
                handle.ui.write(latest.id.replace('%', '%%'))
        return resources

class Images(pluginapi.Plugin):
    name = 'images'
    DEPLOY = 'deploy_image_on_target'
    LAUNCH = 'launch_system_on_target'
    CANCEL = 'image_build_cancellation'

    def _createJob(self, action_type, image_name, target_name, doLaunch):
        rb = self.handle.facade.rbuilder

        project, branch, stage = self._getProductStage()
        query_params = dict(
            project=project,
            branch=branch,
            stage=stage,
            order_by='-time_created',
            )
        if image_name.isdigit():
            query_params['image_id'] = image_name
        else:
            image_name, _, version = image_name.partition('=')
            query_params['name'] = image_name
            if version:
                query_params['trailing_version'] = version

        images = rb.getImages(**query_params)
        if not images:
            raise errors.PluginError("No image matching '%s'" % image_name)

        target = rb.getTargets(name=target_name)
        if not target:
            raise errors.PluginError("No target matching '%s'" % target_name)
        target = target[0]
        if target.is_configured == 'false':
            raise errors.PluginError(("Target '{0}' is not configured. Try"
                " running \"rbuild edit target '{0}'\" or contacting"
                " your rbuilder administrator.").format(target.name))

        if target.credentials_valid == 'false':
            raise errors.PluginError(("Target '{0}' does not have valid"
                " credentials. Try running \"rbuild edit target '{0}'\""
                " and updating your credentials.").format(target.name))

        image, action = self._getAction(images, target, action_type)

        ddata = self.handle.DescriptorConfig.createDescriptorData(
            fromStream=action.descriptor)

        doc = xobj.Document()
        doc.job = job = xobj.XObj()

        job.job_type = action._root.job_type
        job.descriptor = action._root.descriptor
        job.descriptor_data = xobj.parse(ddata.toxml()).descriptor_data

        if doLaunch:
            return image.jobs.append(doc)

    def _getAction(self, images, target, key):
        assert key in (self.DEPLOY, self.LAUNCH)

        for image in images:
            if image.status != '300':
                continue
            for action in image.actions:
                if key == action.key and target.name in action.name:
                    return image, action
        raise errors.PluginError(
            "cannot %s %s" % (key.replace('_', ' '), target))

    def _getProductStage(self):
        try:
            product = self.handle.product.getProductShortname()
            baseLabel = self.handle.product.getBaseLabel()
        except AttributeError:
            raise errors.MissingProductStoreError(path=os.getcwd())

        try:
            stage = self.handle.productStore.getActiveStageName()
        except errors.RbuildError:
            raise errors.MissingActiveStageError(path=os.getcwd())
        return (product, baseLabel, stage)

    def cancel(self, image):
        '''
        Cancel a currently running image build

        :param image: image obj
        :type image: rObj(image)
        '''
        if image.status != '100':
            raise CancelImageError(msg="Image '%s' is not currently building" %
                image.image_id)

        cancelAction = [a for a in image.actions if a.key == self.CANCEL]
        if not cancelAction:
            raise CancelImageError(msg="Unable to find cancel action for"
                " image '%s'" % image.image_id)
        cancelAction = cancelAction[0]
        ddata = self.handle.DescriptorConfig.createDescriptorData(
            fromStream=cancelAction.descriptor)
        doc = xobj.Document()
        doc.job = job = xobj.XObj()

        job.job_type = cancelAction._root.job_type
        job.descriptor = cancelAction._root.descriptor
        job.descriptor_data = xobj.parse(ddata.toxml()).descriptor_data
        return image.jobs.append(doc)

    def deployImage(self, *args, **kwargs):
        '''
        Deploys an image template to a target

        @param image: name of image to deploy
        @type image: str
        @param target: name of target to deploy to
        @type target: str
        @param config: deploy configuration data
        @type config: dict
        @return: image deploy job
        @rtype: rObj(job)
        '''
        return self._createJob(self.DEPLOY, *args, **kwargs)

    def delete(self, imageId):
        shortName, baseLabel, stageName = self._getProductStage()

        images = self.handle.facade.rbuilder.getImages(image_id=imageId,
            project=shortName, branch=baseLabel, stage=stageName)
        if images:
            images[0].delete()
        else:
            raise MissingImageError(image=imageId, project=shortName,
                stage=stageName)

    def launchImage(self, *args, **kwargs):
        '''
        Launches an image to a target

        @param image: name of image to deploy
        @type image: str
        @param target: name of target to launch to
        @type target: str
        @param config: deploy configuration data
        @type config: dict
        @return: image launch job
        @rtype: rObj(job)
        '''
        return self._createJob(self.LAUNCH, *args, **kwargs)

    def list(self):
        shortName, baseLabel, stageName = self._getProductStage()
        images = self.handle.facade.rbuilder.getImages(project=shortName,
            stage=stageName, branch=baseLabel)
        return images

    def initialize(self):
        for command, subcommand, commandClass in (
                ('cancel', 'images', CancelImagesCommand),
                ('delete', 'images', DeleteImagesCommand),
                ('list', 'images', ListImagesCommand),
                ):
            cmd = self.handle.Commands.getCommandClass(command)
            cmd.registerSubCommand(subcommand, commandClass)

    def registerCommands(self):
        self.handle.Commands.registerCommand(LaunchCommand)

    def show(self, imageId):
        '''
            Show details of a specific image

            @param imageId: id of image
            @type imageId: str or int
        '''
        shortName, baseLabel, stageName = self._getProductStage()
        image = self.handle.facade.rbuilder.getImages(image_id=imageId,
            project=shortName, branch=baseLabel, stage=stageName)
        if image:
            return image[0]

    def watchJob(self, job):
        last_status = None
        while job.job_state.name in ['Queued', 'Running']:
            status = job.status_text
            if status != last_status:
                self.handle.ui.lineOutProgress(status.replace('%', '%%'))
            last_status = status
            time.sleep(1)
            job.refresh()

        if job.job_state.name == 'Failed':
            raise errors.PluginError(job.status_text)

        if self.handle.ui.outStream.isatty():
            self.handle.ui.write()

        if job.job_type.name.startswith('launch'):
            for resource in job.created_resources:
                if hasattr(resource, 'networks'):
                    msg = 'Created system %s with address' % resource.name
                    if len(resource.networks) > 1:
                        msg += 'es'
                    msg += ': '
                    msg += ', '.join(n.dns_name for n in resource.networks)
                    self.handle.ui.write(msg)
<commit_msg>Add trove version to 'list images' output<commit_after>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

'''
images
'''
import os
import time

from xobj import xobj

from rbuild import errors
from rbuild import pluginapi
from rbuild.pluginapi import command

class CancelImageError(errors.RbuildError):
    """Raised when there is an error canceling an image build"""

class MissingImageError(errors.RbuildError):
    """Raised when we don"t find an image"""
    template = ("Unable to find image with id '%(image)s' on %(stage)s stage"
        " of %(project)s project")
    params = ["image", "project", "stage"]

class CancelImagesCommand(command.BaseCommand):
    help = 'Cancel image build'
    paramHelp = '<id>+'

    def runCommand(self, handle, argSet, args):
        _, imageIds = self.requireParameters(args, expected=['id'],
            appendExtra=True)

        project, branch, stage = handle.Images._getProductStage()
        for imageId in imageIds:
            try:
                int(imageId)
            except ValueError:
                raise errors.BadParameterError(
                    "Cannot parse image id '%s'" % imageId)

            image = handle.facade.rbuilder.getImages(image_id=imageId,
                project=project, branch=branch, stage=stage)

            if not image:
                handle.ui.warning(str(MissingImageError(image=imageId,
                    project=project, stage=stage)))
                continue
            image = image[0]

            try:
                handle.Images.cancel(image)
            except CancelImageError as err:
                handle.ui.warning(str(err))

class DeleteImagesCommand(command.BaseCommand):
    help = 'Delete images'
    paramHelp = '<image id>+'

    def runCommand(self, handle, argSet, args):
        _, imageIds = self.requireParameters(
            args, expected=['IMAGEID'], appendExtra=True)
        for imageId in imageIds:
            try:
                int(imageId)
                handle.Images.delete(imageId)
            except ValueError:
                handle.ui.warning("Cannot parse image id '%s'" % imageId)

class LaunchCommand(command.BaseCommand):
    help = 'Launch/Deploy an image onto a target'
    paramHelp = '<image name[=version] | image id> <target name>'
    commands = ['launch', 'deploy']
    docs = {'list': 'List available targets',
            'from-file': 'Load launch/deploy descriptor from file',
            'to-file': 'Write launch/deploy descriptor to file',
            'no-watch': 'Do not wait for job to complete',
            'no-launch': 'Do not launch an image',
            }

    def addLocalParameters(self, argDef):
        argDef['list'] = '-l', command.NO_PARAM
        argDef['from-file'] = '-f', command.ONE_PARAM
        argDef['to-file'] = '-o', command.ONE_PARAM
        argDef['no-watch'] = command.NO_PARAM
        argDef['no-launch'] = command.NO_PARAM

    def runCommand(self, handle, argSet, args):
        ui = handle.ui
        rb = handle.facade.rbuilder

        listTargets = argSet.pop('list', False)
        fromFile = argSet.pop('from-file', None)
        toFile = argSet.pop('to-file', None)
        watch = not argSet.pop('no-watch', False)
        doLaunch = not argSet.pop('no-launch', False)

        if listTargets:
            ui.write('Available targets: %s' %
                     ', '.join(t.name for t in rb.getEnabledTargets()))
            return

        if fromFile:
            handle.DescriptorConfig.readConfig(fromFile)

        command, image, target = self.requireParameters(
            args, expected=['image name or id', 'target name'])

        if command == 'deploy':
            job = handle.Images.deployImage(image, target, doLaunch)
        else:
            job = handle.Images.launchImage(image, target, doLaunch)

        if watch and doLaunch:
            handle.Images.watchJob(job)

        if toFile:
            handle.DescriptorConfig.writeConfig(toFile)

class ListImagesCommand(command.ListCommand):
    help = 'list images'
    resource = 'images'
    listFields = ('image_id', 'name', 'image_type', 'architecture',
        'trailing_version', 'status', 'status_message',)
    listFieldMap = dict(
        image_type=dict(accessor=lambda i: i.image_type.name),
        trailing_version=dict(display_name='Version'),
        )
    showFieldMap = dict(
        actions=dict(hidden=True),
        build_log=dict(accessor=lambda i: i._root.build_log.id),
        created_by=dict(accessor=lambda i: i.created_by.full_name),
        files=dict(
            accessor=lambda i: ', '.join('%s: %s' % (f.title, f.url)
                                         for f in i.files),
            ),
        jobs=dict(accessor=lambda i: i._root.jobs.id),
        project=dict(accessor=lambda i: i.project.name),
        project_branch=dict(accessor=lambda i: i.project_branch.name[0]),
        # add in fields definied in listFieldMap
        **listFieldMap
        )

    def _list(self, handle, *args, **kwargs):
        resources = super(ListImagesCommand, self)._list(
            handle, *args, **kwargs)
        if resources:
            handle.ui.write('\nLatest:')
            for latest in resources._node.latest_files:
                handle.ui.write(latest.id.replace('%', '%%'))
        return resources

class Images(pluginapi.Plugin):
    name = 'images'
    DEPLOY = 'deploy_image_on_target'
    LAUNCH = 'launch_system_on_target'
    CANCEL = 'image_build_cancellation'

    def _createJob(self, action_type, image_name, target_name, doLaunch):
        rb = self.handle.facade.rbuilder

        project, branch, stage = self._getProductStage()
        query_params = dict(
            project=project,
            branch=branch,
            stage=stage,
            order_by='-time_created',
            )
        if image_name.isdigit():
            query_params['image_id'] = image_name
        else:
            image_name, _, version = image_name.partition('=')
            query_params['name'] = image_name
            if version:
                query_params['trailing_version'] = version

        images = rb.getImages(**query_params)
        if not images:
            raise errors.PluginError("No image matching '%s'" % image_name)

        target = rb.getTargets(name=target_name)
        if not target:
            raise errors.PluginError("No target matching '%s'" % target_name)
        target = target[0]
        if target.is_configured == 'false':
            raise errors.PluginError(("Target '{0}' is not configured. Try"
                " running \"rbuild edit target '{0}'\" or contacting"
                " your rbuilder administrator.").format(target.name))

        if target.credentials_valid == 'false':
            raise errors.PluginError(("Target '{0}' does not have valid"
                " credentials. Try running \"rbuild edit target '{0}'\""
                " and updating your credentials.").format(target.name))

        image, action = self._getAction(images, target, action_type)

        ddata = self.handle.DescriptorConfig.createDescriptorData(
            fromStream=action.descriptor)

        doc = xobj.Document()
        doc.job = job = xobj.XObj()

        job.job_type = action._root.job_type
        job.descriptor = action._root.descriptor
        job.descriptor_data = xobj.parse(ddata.toxml()).descriptor_data

        if doLaunch:
            return image.jobs.append(doc)

    def _getAction(self, images, target, key):
        assert key in (self.DEPLOY, self.LAUNCH)

        for image in images:
            if image.status != '300':
                continue
            for action in image.actions:
                if key == action.key and target.name in action.name:
                    return image, action
        raise errors.PluginError(
            "cannot %s %s" % (key.replace('_', ' '), target))

    def _getProductStage(self):
        try:
            product = self.handle.product.getProductShortname()
            baseLabel = self.handle.product.getBaseLabel()
        except AttributeError:
            raise errors.MissingProductStoreError(path=os.getcwd())

        try:
            stage = self.handle.productStore.getActiveStageName()
        except errors.RbuildError:
            raise errors.MissingActiveStageError(path=os.getcwd())
        return (product, baseLabel, stage)

    def cancel(self, image):
        '''
        Cancel a currently running image build

        :param image: image obj
        :type image: rObj(image)
        '''
        if image.status != '100':
            raise CancelImageError(msg="Image '%s' is not currently building" %
                image.image_id)

        cancelAction = [a for a in image.actions if a.key == self.CANCEL]
        if not cancelAction:
            raise CancelImageError(msg="Unable to find cancel action for"
                " image '%s'" % image.image_id)
        cancelAction = cancelAction[0]
        ddata = self.handle.DescriptorConfig.createDescriptorData(
            fromStream=cancelAction.descriptor)
        doc = xobj.Document()
        doc.job = job = xobj.XObj()

        job.job_type = cancelAction._root.job_type
        job.descriptor = cancelAction._root.descriptor
        job.descriptor_data = xobj.parse(ddata.toxml()).descriptor_data
        return image.jobs.append(doc)

    def deployImage(self, *args, **kwargs):
        '''
        Deploys an image template to a target

        @param image: name of image to deploy
        @type image: str
        @param target: name of target to deploy to
        @type target: str
        @param config: deploy configuration data
        @type config: dict
        @return: image deploy job
        @rtype: rObj(job)
        '''
        return self._createJob(self.DEPLOY, *args, **kwargs)

    def delete(self, imageId):
        shortName, baseLabel, stageName = self._getProductStage()

        images = self.handle.facade.rbuilder.getImages(image_id=imageId,
            project=shortName, branch=baseLabel, stage=stageName)
        if images:
            images[0].delete()
        else:
            raise MissingImageError(image=imageId, project=shortName,
                stage=stageName)

    def launchImage(self, *args, **kwargs):
        '''
        Launches an image to a target

        @param image: name of image to deploy
        @type image: str
        @param target: name of target to launch to
        @type target: str
        @param config: deploy configuration data
        @type config: dict
        @return: image launch job
        @rtype: rObj(job)
        '''
        return self._createJob(self.LAUNCH, *args, **kwargs)

    def list(self):
        shortName, baseLabel, stageName = self._getProductStage()
        images = self.handle.facade.rbuilder.getImages(project=shortName,
            stage=stageName, branch=baseLabel)
        return images

    def initialize(self):
        for command, subcommand, commandClass in (
                ('cancel', 'images', CancelImagesCommand),
                ('delete', 'images', DeleteImagesCommand),
                ('list', 'images', ListImagesCommand),
                ):
            cmd = self.handle.Commands.getCommandClass(command)
            cmd.registerSubCommand(subcommand, commandClass)

    def registerCommands(self):
        self.handle.Commands.registerCommand(LaunchCommand)

    def show(self, imageId):
        '''
            Show details of a specific image

            @param imageId: id of image
            @type imageId: str or int
        '''
        shortName, baseLabel, stageName = self._getProductStage()
        image = self.handle.facade.rbuilder.getImages(image_id=imageId,
            project=shortName, branch=baseLabel, stage=stageName)
        if image:
            return image[0]

    def watchJob(self, job):
        last_status = None
        while job.job_state.name in ['Queued', 'Running']:
            status = job.status_text
            if status != last_status:
                self.handle.ui.lineOutProgress(status.replace('%', '%%'))
            last_status = status
            time.sleep(1)
            job.refresh()

        if job.job_state.name == 'Failed':
            raise errors.PluginError(job.status_text)

        if self.handle.ui.outStream.isatty():
            self.handle.ui.write()

        if job.job_type.name.startswith('launch'):
            for resource in job.created_resources:
                if hasattr(resource, 'networks'):
                    msg = 'Created system %s with address' % resource.name
                    if len(resource.networks) > 1:
                        msg += 'es'
                    msg += ': '
                    msg += ', '.join(n.dns_name for n in resource.networks)
                    self.handle.ui.write(msg)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Exports anything passed in as ISRI Extended ASCII, a Latin-1 based encoding
which allows for Unicode BMP escapes. Astral code points are impossible to
express in this scheme, lest one wishes to resort to surrogate pairs (yuck!).
"""

import os
import re
import sys
import fileinput
import codecs

__author__ = 'Eddie Antonio Santos'
__version__ = '1.0.0'
__license__ = 'MIT'

# Set of ASCII and Latin-1 characters.
PRINTABLE  = set(unichr(codepoint) for codepoint in range(0x20, 0x7F))
PRINTABLE |= set(unichr(codepoint) for codepoint in range(0xA1, 0x100))

def to_extended_ascii(character):
    ur"""
    >>> print(to_extended_ascii('A'))
    A
    >>> len(to_extended_ascii('Ä'))
    1
    >>> print(to_extended_ascii('Ω'))
    <03A9>
    >>> print(to_extended_ascii('א'))
    <05D0>
    >>> print(to_extended_ascii('\n'))
    <\n>
    >>> to_extended_ascii('🎎')
    Traceback (most recent call last):
    ...
    ValueError: Cannot encode astral code point
    """

    if not isinstance(character, unicode):
        character = unicode(character, 'UTF-8')
    if len(character.encode('UTF-32LE')) != 4:
        raise ValueError('Must be exactly one BMP code point')

    try:
        codepoint = ord(character)
    except TypeError:
        raise ValueError('Cannot encode astral code point')

    if character == '\n':
        # Newlines are given a special escape.
        return ur'<\n>'
    elif codepoint >= 0x100:
        # Use <xxxx> for UCS-2.
        return u'<%04X>' % (codepoint,)
    elif character in PRINTABLE:
        # Any printable Latin-1 character is returned verbatim.
        return character
    else:
        assert codepoint < 0x100
        # Use <xx> for control characters.
        return u'<%02X>' % (codepoint,)

def extended_ascii():
    # Open stdout in binary mode. Let's write some raw bytes!
    stdout = os.fdopen(sys.stdout.fileno(), 'wb')
    sys.stdout.close()

    # For every unicode character in the input...
    for line in fileinput.input():
        for char in unicode(line, 'UTF-8'):
            # Convert it to extended ASCII
            byte_string = to_extended_ascii(char).encode('latin-1')
            stdout.write(byte_string)

def unextend(text):
    ur"""
    >>> print(unextend(r"A<\n><03A9><05D0>"))
    A
    Ωא
    """
    def replacement(match):
        code_point, = match.groups()
        return unichr(int(code_point, base=16))

    return re.sub('<([0-9A-F]{2,4})>', replacement, text).replace(r'<\n>', '\n')

def to_utf8():
    for line in fileinput.input():
        text = unicode(line, 'latin-1')
        print(unextend(text))

def main():
    program_name = os.path.basename(sys.argv[0])
    if program_name == 'to_utf8':
        to_utf8()
    else:
        extended_ascii()

if __name__ == '__main__':
    main()
<commit_msg>Fix trailing newline in to_utf8.<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Exports anything passed in as ISRI Extended ASCII, a Latin-1 based encoding
which allows for Unicode BMP escapes. Astral code points are impossible to
express in this scheme, lest one wishes to resort to surrogate pairs (yuck!).
"""

import os
import re
import sys
import fileinput
import codecs

__author__ = 'Eddie Antonio Santos'
__version__ = '1.0.0'
__license__ = 'MIT'

# Set of ASCII and Latin-1 characters.
PRINTABLE  = set(unichr(codepoint) for codepoint in range(0x20, 0x7F))
PRINTABLE |= set(unichr(codepoint) for codepoint in range(0xA1, 0x100))

def to_extended_ascii(character):
    ur"""
    >>> print(to_extended_ascii('A'))
    A
    >>> len(to_extended_ascii('Ä'))
    1
    >>> print(to_extended_ascii('Ω'))
    <03A9>
    >>> print(to_extended_ascii('א'))
    <05D0>
    >>> print(to_extended_ascii('\n'))
    <\n>
    >>> to_extended_ascii('🎎')
    Traceback (most recent call last):
    ...
    ValueError: Cannot encode astral code point
    """

    if not isinstance(character, unicode):
        character = unicode(character, 'UTF-8')
    if len(character.encode('UTF-32LE')) != 4:
        raise ValueError('Must be exactly one BMP code point')

    try:
        codepoint = ord(character)
    except TypeError:
        raise ValueError('Cannot encode astral code point')

    if character == '\n':
        # Newlines are given a special escape.
        return ur'<\n>'
    elif codepoint >= 0x100:
        # Use <xxxx> for UCS-2.
        return u'<%04X>' % (codepoint,)
    elif character in PRINTABLE:
        # Any printable Latin-1 character is returned verbatim.
        return character
    else:
        assert codepoint < 0x100
        # Use <xx> for control characters.
        return u'<%02X>' % (codepoint,)

def extended_ascii():
    # Open stdout in binary mode. Let's write some raw bytes!
    stdout = os.fdopen(sys.stdout.fileno(), 'wb')
    sys.stdout.close()

    # For every unicode character in the input...
    for line in fileinput.input():
        for char in unicode(line, 'UTF-8'):
            # Convert it to extended ASCII
            byte_string = to_extended_ascii(char).encode('latin-1')
            stdout.write(byte_string)

def unextend(text):
    ur"""
    >>> print(unextend(r"A<\n><03A9><05D0>"))
    A
    Ωא
    """
    def replacement(match):
        code_point, = match.groups()
        return unichr(int(code_point, base=16))

    return re.sub('<([0-9A-F]{2,4})>', replacement, text).replace(r'<\n>', '\n')

def to_utf8():
    for line in fileinput.input():
        text = unicode(line, 'latin-1')
        sys.stdout.write(unextend(text))

def main():
    program_name = os.path.basename(sys.argv[0])
    if program_name == 'to_utf8':
        to_utf8()
    else:
        extended_ascii()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#
#   This file is part of m.css.
#
#   Copyright © 2017, 2018, 2019 Vladimír Vondruš <mosra@centrum.cz>
#
#   Permission is hereby granted, free of charge, to any person obtaining a
#   copy of this software and associated documentation files (the "Software"),
#   to deal in the Software without restriction, including without limitation
#   the rights to use, copy, modify, merge, publish, distribute, sublicense,
#   and/or sell copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following conditions:
#
#   The above copyright notice and this permission notice shall be included
#   in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#   THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#   DEALINGS IN THE SOFTWARE.
#

import copy
import html
import os
import re

from docutils import nodes, utils
from docutils.parsers import rst
from docutils.parsers.rst import directives
from docutils.parsers.rst.roles import set_classes

import pelican.signals

import latex2svg
import latex2svgextra

default_settings = {
    'INPUT': '',
    'M_MATH_RENDER_AS_CODE': False,
    'M_MATH_CACHE_FILE': 'm.math.cache'
}

settings = None

def _is_math_figure(parent):
    # The parent has to be a figure, marked as m-figure
    if not isinstance(parent, nodes.figure): return False
    if 'm-figure' not in parent.get('classes', []): return False

    # And as a first visible node of such type
    for child in parent:
        if not isinstance(child, nodes.Invisible): return False

    return True

class Math(rst.Directive):
    option_spec = {'class': directives.class_option,
                   'name': directives.unchanged}
    has_content = True

    def run(self):
        set_classes(self.options)
        self.assert_has_content()

        parent = self.state.parent

        # Fallback rendering as code requested
        if settings['M_MATH_RENDER_AS_CODE']:
            # If this is a math figure, replace the figure CSS class to have a
            # matching border
            if _is_math_figure(parent):
                parent['classes'][parent['classes'].index('m-figure')] = 'm-code-figure'

            content = nodes.raw('', html.escape('\n'.join(self.content)), format='html')
            pre = nodes.literal_block('')
            pre.append(content)
            return [pre]

        content = '\n'.join(self.content)

        _, svg = latex2svgextra.fetch_cached_or_render("$$" + content + "$$")

        # If this is the first real node inside a math figure, put the SVG
        # directly inside
        if _is_math_figure(parent):
            node = nodes.raw(self.block_text, latex2svgextra.patch(content, svg, None, ' class="{}"'.format(' '.join(['m-math'] + self.options.get('classes', [])))), format='html')
            node.line = self.content_offset + 1
            self.add_name(node)
            return [node]

        # Otherwise wrap it in a

        node = nodes.raw(self.block_text, latex2svgextra.patch(content, svg, None, ''), format='html')
        node.line = self.content_offset + 1
        self.add_name(node)
        container = nodes.container(**self.options)
        container['classes'] += ['m-math']
        container.append(node)
        return [container]

def new_page(*args):
    latex2svgextra.counter = 0

def math(role, rawtext, text, lineno, inliner, options={}, content=[]):
    # Otherwise the backslashes do quite a mess there
    i = rawtext.find('`')
    text = rawtext.split('`')[1]

    # Fallback rendering as code requested
    if settings['M_MATH_RENDER_AS_CODE']:
        set_classes(options)
        classes = []
        if 'classes' in options:
            classes += options['classes']
            del options['classes']

        content = nodes.raw('', html.escape(utils.unescape(text)), format='html')
        node = nodes.literal(rawtext, '', **options)
        node.append(content)
        return [node], []

    # Apply classes to the <svg> element instead of some outer
    set_classes(options)
    classes = 'm-math'
    if 'classes' in options:
        classes += ' ' + ' '.join(options['classes'])
        del options['classes']

    depth, svg = latex2svgextra.fetch_cached_or_render("$" + text + "$")

    attribs = ' class="{}"'.format(classes)
    node = nodes.raw(rawtext, latex2svgextra.patch(text, svg, depth, attribs), format='html', **options)
    return [node], []

def save_cache(*args):
    if settings['M_MATH_CACHE_FILE']:
        latex2svgextra.pickle_cache(settings['M_MATH_CACHE_FILE'])

def register_mcss(mcss_settings, hooks_pre_page, hooks_post_run, **kwargs):
    global default_settings, settings
    settings = copy.deepcopy(default_settings)
    for key in settings.keys():
        if key in mcss_settings: settings[key] = mcss_settings[key]

    if settings['M_MATH_CACHE_FILE']:
        settings['M_MATH_CACHE_FILE'] = os.path.join(settings['INPUT'], settings['M_MATH_CACHE_FILE'])

        if os.path.exists(settings['M_MATH_CACHE_FILE']):
            latex2svgextra.unpickle_cache(settings['M_MATH_CACHE_FILE'])
        else:
            latex2svgextra.unpickle_cache(None)

    hooks_pre_page += [new_page]
    hooks_post_run += [save_cache]

    rst.directives.register_directive('math', Math)
    rst.roles.register_canonical_role('math', math)

def _configure_pelican(pelicanobj):
    register_mcss(mcss_settings=pelicanobj.settings, hooks_pre_page=[], hooks_post_run=[])

def register():
    pelican.signals.initialized.connect(_configure_pelican)
    pelican.signals.finalized.connect(save_cache)
    pelican.signals.content_object_init.connect(new_page)
<commit_msg>m.math: ensure cache file is reset when cache is not used.<commit_after>#
#   This file is part of m.css.
#
#   Copyright © 2017, 2018, 2019 Vladimír Vondruš <mosra@centrum.cz>
#
#   Permission is hereby granted, free of charge, to any person obtaining a
#   copy of this software and associated documentation files (the "Software"),
#   to deal in the Software without restriction, including without limitation
#   the rights to use, copy, modify, merge, publish, distribute, sublicense,
#   and/or sell copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following conditions:
#
#   The above copyright notice and this permission notice shall be included
#   in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#   THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#   DEALINGS IN THE SOFTWARE.
#

import copy
import html
import os
import re

from docutils import nodes, utils
from docutils.parsers import rst
from docutils.parsers.rst import directives
from docutils.parsers.rst.roles import set_classes

import pelican.signals

import latex2svg
import latex2svgextra

default_settings = {
    'INPUT': '',
    'M_MATH_RENDER_AS_CODE': False,
    'M_MATH_CACHE_FILE': 'm.math.cache'
}

settings = None

def _is_math_figure(parent):
    # The parent has to be a figure, marked as m-figure
    if not isinstance(parent, nodes.figure): return False
    if 'm-figure' not in parent.get('classes', []): return False

    # And as a first visible node of such type
    for child in parent:
        if not isinstance(child, nodes.Invisible): return False

    return True

class Math(rst.Directive):
    option_spec = {'class': directives.class_option,
                   'name': directives.unchanged}
    has_content = True

    def run(self):
        set_classes(self.options)
        self.assert_has_content()

        parent = self.state.parent

        # Fallback rendering as code requested
        if settings['M_MATH_RENDER_AS_CODE']:
            # If this is a math figure, replace the figure CSS class to have a
            # matching border
            if _is_math_figure(parent):
                parent['classes'][parent['classes'].index('m-figure')] = 'm-code-figure'

            content = nodes.raw('', html.escape('\n'.join(self.content)), format='html')
            pre = nodes.literal_block('')
            pre.append(content)
            return [pre]

        content = '\n'.join(self.content)

        _, svg = latex2svgextra.fetch_cached_or_render("$$" + content + "$$")

        # If this is the first real node inside a math figure, put the SVG
        # directly inside
        if _is_math_figure(parent):
            node = nodes.raw(self.block_text, latex2svgextra.patch(content, svg, None, ' class="{}"'.format(' '.join(['m-math'] + self.options.get('classes', [])))), format='html')
            node.line = self.content_offset + 1
            self.add_name(node)
            return [node]

        # Otherwise wrap it in a

        node = nodes.raw(self.block_text, latex2svgextra.patch(content, svg, None, ''), format='html')
        node.line = self.content_offset + 1
        self.add_name(node)
        container = nodes.container(**self.options)
        container['classes'] += ['m-math']
        container.append(node)
        return [container]

def new_page(*args):
    latex2svgextra.counter = 0

def math(role, rawtext, text, lineno, inliner, options={}, content=[]):
    # Otherwise the backslashes do quite a mess there
    i = rawtext.find('`')
    text = rawtext.split('`')[1]

    # Fallback rendering as code requested
    if settings['M_MATH_RENDER_AS_CODE']:
        set_classes(options)
        classes = []
        if 'classes' in options:
            classes += options['classes']
            del options['classes']

        content = nodes.raw('', html.escape(utils.unescape(text)), format='html')
        node = nodes.literal(rawtext, '', **options)
        node.append(content)
        return [node], []

    # Apply classes to the <svg> element instead of some outer
    set_classes(options)
    classes = 'm-math'
    if 'classes' in options:
        classes += ' ' + ' '.join(options['classes'])
        del options['classes']

    depth, svg = latex2svgextra.fetch_cached_or_render("$" + text + "$")

    attribs = ' class="{}"'.format(classes)
    node = nodes.raw(rawtext, latex2svgextra.patch(text, svg, depth, attribs), format='html', **options)
    return [node], []

def save_cache(*args):
    if settings['M_MATH_CACHE_FILE']:
        latex2svgextra.pickle_cache(settings['M_MATH_CACHE_FILE'])

def register_mcss(mcss_settings, hooks_pre_page, hooks_post_run, **kwargs):
    global default_settings, settings
    settings = copy.deepcopy(default_settings)
    for key in settings.keys():
        if key in mcss_settings: settings[key] = mcss_settings[key]

    if settings['M_MATH_CACHE_FILE']:
        settings['M_MATH_CACHE_FILE'] = os.path.join(settings['INPUT'], settings['M_MATH_CACHE_FILE'])

    # Ensure that cache is unpickled again if M_MATH_CACHE_FILE is *not* set --
    # otherwise tests will sporadically fail.
    if settings['M_MATH_CACHE_FILE'] and os.path.exists(settings['M_MATH_CACHE_FILE']):
        latex2svgextra.unpickle_cache(settings['M_MATH_CACHE_FILE'])
    else:
        latex2svgextra.unpickle_cache(None)

    hooks_pre_page += [new_page]
    hooks_post_run += [save_cache]

    rst.directives.register_directive('math', Math)
    rst.roles.register_canonical_role('math', math)

def _configure_pelican(pelicanobj):
    register_mcss(mcss_settings=pelicanobj.settings, hooks_pre_page=[], hooks_post_run=[])

def register():
    pelican.signals.initialized.connect(_configure_pelican)
    pelican.signals.finalized.connect(save_cache)
    pelican.signals.content_object_init.connect(new_page)
<|endoftext|>


<commit_before>import os

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import WindowsAPIError
from pywincffi.kernel32.io import CloseHandle
from pywincffi.kernel32.process import (
    OpenProcess, GetCurrentProcess, GetProcessId)

class TestOpenProcess(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.process.OpenProcess`
    """
    def test_returns_handle(self):
        ffi, library = dist.load()

        handle = OpenProcess(
            library.PROCESS_QUERY_INFORMATION,
            False,
            os.getpid()
        )

        typeof = ffi.typeof(handle)
        self.assertEqual(typeof.kind, "pointer")
        self.assertEqual(typeof.cname, "void *")
        CloseHandle(handle)

    def test_access_denied_for_null_desired_access(self):
        with self.assertRaises(WindowsAPIError) as error:
            OpenProcess(0, False, os.getpid())

        self.assertEqual(error.exception.code, 5)

class TestGetCurrentProcess(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.process.GetCurrentProcess`
    """
    def test_returns_handle(self):
        ffi, library = dist.load()
        handle = GetCurrentProcess()
        typeof = ffi.typeof(handle)
        self.assertEqual(typeof.kind, "pointer")
        self.assertEqual(typeof.cname, "void *")

    def test_returns_same_handle(self):
        # GetCurrentProcess is somewhat special in that it will
        # always return a handle to the same object.  However, __eq__ is not
        # opaque so the string representation of the two handles
        # should always match since it contains the address of the object
        # in memory.
        self.assertEqual(repr(GetCurrentProcess()), repr(GetCurrentProcess()))

    def test_handle_is_current_process(self):
        handle = GetCurrentProcess()
        self.assertEqual(GetProcessId(handle), os.getpid())

    def test_handle_is_valid(self):
        _, library = dist.load()
        handle = OpenProcess(
            library.PROCESS_QUERY_INFORMATION, False, os.getpid())

        # If the handle were invalid, this would fail.
        CloseHandle(handle)
<commit_msg>make sure we can query the pid from the handle we returned from OpenProcess<commit_after>import os

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import WindowsAPIError
from pywincffi.kernel32.io import CloseHandle
from pywincffi.kernel32.process import (
    OpenProcess, GetCurrentProcess, GetProcessId)

class TestOpenProcess(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.process.OpenProcess`
    """
    def test_returns_handle(self):
        ffi, library = dist.load()

        handle = OpenProcess(
            library.PROCESS_QUERY_INFORMATION,
            False,
            os.getpid()
        )

        typeof = ffi.typeof(handle)
        self.assertEqual(typeof.kind, "pointer")
        self.assertEqual(typeof.cname, "void *")
        CloseHandle(handle)

    def test_access_denied_for_null_desired_access(self):
        with self.assertRaises(WindowsAPIError) as error:
            OpenProcess(0, False, os.getpid())

        self.assertEqual(error.exception.code, 5)

    def test_get_process_id_current_process(self):
        # We should be able to access the pid of the process
        # we created a handle to.
        ffi, library = dist.load()

        handle = OpenProcess(
            library.PROCESS_QUERY_INFORMATION,
            False,
            os.getpid()
        )
        self.assertEqual(GetProcessId(handle), os.getpid())

class TestGetCurrentProcess(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.process.GetCurrentProcess`
    """
    def test_returns_handle(self):
        ffi, library = dist.load()
        handle = GetCurrentProcess()
        typeof = ffi.typeof(handle)
        self.assertEqual(typeof.kind, "pointer")
        self.assertEqual(typeof.cname, "void *")

    def test_returns_same_handle(self):
        # GetCurrentProcess is somewhat special in that it will
        # always return a handle to the same object.  However, __eq__ is not
        # opaque so the string representation of the two handles
        # should always match since it contains the address of the object
        # in memory.
        self.assertEqual(repr(GetCurrentProcess()), repr(GetCurrentProcess()))

    def test_handle_is_current_process(self):
        handle = GetCurrentProcess()
        self.assertEqual(GetProcessId(handle), os.getpid())

    def test_handle_is_valid(self):
        _, library = dist.load()
        handle = OpenProcess(
            library.PROCESS_QUERY_INFORMATION, False, os.getpid())

        # If the handle were invalid, this would fail.
        CloseHandle(handle)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.code import Switch
from hwt.hdlObjects.typeShortcuts import vecT
from hwt.interfaces.std import Signal
from hwt.synthesizer.param import Param
from hwtLib.handshaked.compBase import HandshakedCompBase

class HsSplitSelect(HandshakedCompBase):
    """
    Split data from input interface to output interface based on explicit output index
    provided by select interface

    .. aafig::
                *----+ output0
               *     +-------->
         input |     |
        +------>     | output1
               |     +-------->
               *     |
                *-^--+
                  |
                  +
                select
    """
    def _config(self):
        self.OUTPUTS = Param(2)
        super()._config()

    def _declr(self):
        outputs = int(self.OUTPUTS)

        self.sel = Signal(dtype=vecT(outputs.bit_length()))

        with self._paramsShared():
            self.dataIn = self.intfCls()
            self.dataOut = self.intfCls(asArraySize=self.OUTPUTS)

    def _impl(self):
        In = self.dataIn
        rd = self.getRd

        for index, outIntf in enumerate(self.dataOut):
            for ini, outi in zip(In._interfaces, outIntf._interfaces):
                if ini == self.getVld(In):
                    outi ** (ini & self.sel._eq(index))
                elif ini == rd(In):
                    pass
                else:  # data
                    outi ** ini

        Switch(self.sel).addCases(
            [(index, rd(In) ** rd(out))
             for index, out in enumerate(self.dataOut)]
        )

if __name__ == "__main__":
    from hwt.interfaces.std import Handshaked
    from hwt.synthesizer.shortcuts import toRtl
    u = HandshakedDemux(Handshaked)
    print(toRtl(u))
<commit_msg>HsSplitSelect sel input as Handshaked<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.code import Switch
from hwt.synthesizer.param import Param
from hwtLib.handshaked.compBase import HandshakedCompBase

class HsSplitSelect(HandshakedCompBase):
    """
    Split data from input interface to output interface based on explicit output index
    provided by select interface

    .. aafig::
                *----+ output0
               *     +-------->
         input |     |
        +------>     | output1
               |     +-------->
               *     |
                *-^--+
                  |
                  +
                select
    """
    def _config(self):
        self.OUTPUTS = Param(3)
        super()._config()

    def _declr(self):
        outputs = int(self.OUTPUTS)
        assert outputs > 1, outputs

        self.sel = Handshaked()
        self.sel.DATA_WIDTH.set(outputs.bit_length())

        with self._paramsShared():
            self.dataIn = self.intfCls()
            self.dataOut = self.intfCls(asArraySize=self.OUTPUTS)

    def _impl(self):
        In = self.dataIn
        rd = self.getRd
        sel = self.sel

        for index, outIntf in enumerate(self.dataOut):
            for ini, outi in zip(In._interfaces, outIntf._interfaces):
                if ini == self.getVld(In):
                    outi ** (sel.vld & ini & sel.data._eq(index))
                elif ini == rd(In):
                    pass
                else:  # data
                    outi ** ini

        Switch(self.sel.data).addCases(
            [(index, [rd(In) ** rd(out),
                      sel.rd ** (rd(out) & self.getVld(out))])
             for index, out in enumerate(self.dataOut)]
        ).Default(
                  sel.rd ** None,
                  rd(In) ** None
                  )

if __name__ == "__main__":
    from hwt.interfaces.std import Handshaked
    from hwt.synthesizer.shortcuts import toRtl
    u = HsSplitSelect(Handshaked)
    print(toRtl(u))
<|endoftext|>


<commit_before># coding=utf-8

import re
import random

STATE_SIZE = 2
GENERATE_ROUNDS = 3

CHATTINESS = 0.02
ACTIVE_CHANNELS = []

class SqlMark:
    def __init__(self, db, table_name='markov', state_size=3):
        self.db = db
        self.table_name = table_name
        self.state_size = state_size
        self.separator = ' '

        db.execute("""\
            CREATE TABLE IF NOT EXISTS %s(
                tag TEXT COLLATE NOCASE,
                prefix TEXT COLLATE NOCASE,
                next_word TEXT COLLATE NOCASE,
                occurrences INTEGER,
                UNIQUE(tag, prefix, next_word));""" % self.table_name)
        db.commit()

    def add_sentence(self, tag, words):
        words = self.state_size * [''] + words + ['']
        num_words = len(words)
        for i in range(0, num_words - self.state_size, 1):
            prefix = words[i:i + self.state_size]
            next_word = words[i + self.state_size].strip()
            pref_join = self.join(prefix).strip()
            self.db.execute("""\
                INSERT OR IGNORE INTO %s(tag,prefix,next_word,occurrences)
                VALUES(?,?,?,0);""" % self.table_name, (tag, pref_join, next_word))
            self.db.execute("""\
                UPDATE %s SET occurrences = occurrences + 1
                WHERE tag = ? AND prefix = ? AND next_word = ?;""" % self.table_name, (tag, pref_join, next_word))

    def commit(self):
        self.db.commit()

    def join(self, words):
        j = self.separator.join([w for w in words if w != ''])
        return j

    def get_next_word(self, tag, prefix):
        pref_join = self.join(prefix)
        c = self.db.cursor()
        c.execute("SELECT sum(occurrences) FROM %s WHERE tag = ? AND prefix = ?;" % self.table_name, (tag, pref_join))
        max_occ = c.fetchone()[0]
        if max_occ == 0 or max_occ is None:
            return None

        rand = random.randint(0, max_occ)
        n = 0
        c.execute("SELECT next_word, occurrences FROM %s WHERE tag = ? AND prefix = ?;" % self.table_name,
                  (tag, pref_join))
        for row in c:
            n += row[1]
            if n >= rand:
                return None if row[0] == '' else row[0]
        return None

    def generate_sentence(self, tag, start=None, max_words=30):
        result = start
        if result is None:
            result = ['']

        while True:
            next_word = self.get_next_word(tag, result[-self.state_size:])
            if next_word is None or len(result) > max_words:
                return ' '.join(result)
            next_word = next_word.strip()
            result.append(next_word)

m = SqlMark(yui.db, state_size=STATE_SIZE)

def generate(tag, start):
    longest = None
    for i in range(GENERATE_ROUNDS):
        gen = m.generate_sentence(tag, start)
        if longest is None or len(gen) > len(longest):
            longest = gen
    return longest

@yui.admin
@yui.command('markov_loadfile')
def load_file(argv):
    """Add a an IRC log file to the existing markov models. Messages in the file must be the same format as ZNC
    writes them, [NN:NN:NN] <NICK> MSG. Usage: markov_loadfile <path> <channel_name>"""

    if len(argv) < 3:
        return

    try:
        dict_file_name = argv[1]
        with open(dict_file_name, errors='replace') as f:
            text = f.read()
    except:
        return "Couldn't read file"

    msg_regex = re.compile(r'^\[..:..:..\] <(.*?)> (.*)$')

    # go through all lines and extract nick + messages
    num_lines = 0
    for line in text.splitlines():
        match = msg_regex.match(line)
        if not match:
            continue
        nick = match.group(1)
        msg = match.group(2)
        spl = msg.split()
        if len(spl) < 3:
            continue
        m.add_sentence(nick, spl)
        m.add_sentence(argv[2], spl)
        num_lines += 1
    m.commit()
    return 'Loaded %d lines' % num_lines

def contains_mention(split):
    for w in split:
        if w.lower() == yui.get_nick().lower():
            return w
    return ''

# add new sentences as they come in
@yui.event('msgRecv')
def recv(channel, user, msg):
    # train the markov chains
    if channel == user.nick:  # ignore query
        return
    split = msg.split()
    m.add_sentence(user.nick, split)
    m.add_sentence(channel, split)
    m.commit()

    # do chatbot stuff
    if msg.startswith(tuple(yui.config_val('commandPrefixes', default=['!']))) or channel not in ACTIVE_CHANNELS:
        return

    mention = contains_mention(split)
    if mention != '' or random.random() < CHATTINESS:
        if mention != '':
            split.remove(mention)
            split.append(user.nick)
        topic = random.choice(split)
        sentence = generate(channel, [topic])
        if sentence != '':
            yui.send_msg(channel, yui.unhighlight_for_channel(sentence, channel))

@yui.command('markov', 'mark')
def markov(argv, user, channel):
    """Generate a random sentence for a given nick. Usage: markov [nick] [sentence_start]"""
    name = user.nick
    sentence_start = ['']
    if len(argv) > 1:
        name = argv[1]
    if len(argv) > 2:
        sentence_start = argv[2:2+STATE_SIZE]

    sentence = generate(name, sentence_start)
    if sentence == '':
        return "Couldn't generate a sentence :("
    return yui.unhighlight_for_channel('<%s> %s' % (name, sentence), channel)

@yui.command('chatbot')
def chatbot(channel, argv):
    """De-/activate random chatbot for a channel. Usage: chatbot [channel]"""
    global ACTIVE_CHANNELS
    if len(argv) > 1:
        channel = argv[1]

    if channel in ACTIVE_CHANNELS:
        ACTIVE_CHANNELS.remove(channel)
        return "I'll shut up"
    ACTIVE_CHANNELS.append(channel)
    return 'Chatting now'
<commit_msg>fix some whitespaces and behaviour on non-matching sentence start<commit_after># coding=utf-8

import re
import random

STATE_SIZE = 2
GENERATE_ROUNDS = 3

CHATTINESS = 0.02
ACTIVE_CHANNELS = []

class SqlMark:
    def __init__(self, db, table_name='markov', state_size=3):
        self.db = db
        self.table_name = table_name
        self.state_size = state_size
        self.separator = ' '

        db.execute("""\
            CREATE TABLE IF NOT EXISTS %s(
                tag TEXT COLLATE NOCASE,
                prefix TEXT COLLATE NOCASE,
                next_word TEXT COLLATE NOCASE,
                occurrences INTEGER,
                UNIQUE(tag, prefix, next_word));""" % self.table_name)
        db.commit()

    def add_sentence(self, tag, words):
        words = self.state_size * [''] + words + ['']
        num_words = len(words)
        for i in range(0, num_words - self.state_size, 1):
            prefix = words[i:i + self.state_size]
            next_word = words[i + self.state_size].strip()
            pref_join = self.join(prefix).strip()
            self.db.execute("""\
                INSERT OR IGNORE INTO %s(tag,prefix,next_word,occurrences)
                VALUES(?,?,?,0);""" % self.table_name, (tag, pref_join, next_word))
            self.db.execute("""\
                UPDATE %s SET occurrences = occurrences + 1
                WHERE tag = ? AND prefix = ? AND next_word = ?;""" % self.table_name, (tag, pref_join, next_word))

    def commit(self):
        self.db.commit()

    def join(self, words):
        j = self.separator.join([w for w in words if w != ''])
        return j

    def get_next_word(self, tag, prefix):
        pref_join = self.join(prefix)
        c = self.db.cursor()
        c.execute("SELECT sum(occurrences) FROM %s WHERE tag = ? AND prefix = ?;" % self.table_name, (tag, pref_join))
        max_occ = c.fetchone()[0]
        if max_occ == 0 or max_occ is None:
            return None

        rand = random.randint(0, max_occ)
        n = 0
        c.execute("SELECT next_word, occurrences FROM %s WHERE tag = ? AND prefix = ?;" % self.table_name,
                  (tag, pref_join))
        for row in c:
            n += row[1]
            if n >= rand:
                return None if row[0] == '' else row[0]
        return None

    def generate_sentence(self, tag, start=None, max_words=30):
        result = start
        if result is None:
            result = ['']

        while True:
            next_word = self.get_next_word(tag, result[-self.state_size:])
            if next_word is None:
                break
            next_word = next_word.strip()
            result.append(next_word)
            if len(result) >= max_words:
                break

        if len(result) > start:
            return ' '.join(result).strip()
        else:
            return ''

m = SqlMark(yui.db, state_size=STATE_SIZE)

def generate(tag, start):
    longest = None
    for i in range(GENERATE_ROUNDS):
        gen = m.generate_sentence(tag, start)
        if longest is None or len(gen) > len(longest):
            longest = gen
    return longest

@yui.admin
@yui.command('markov_loadfile')
def load_file(argv):
    """Add a an IRC log file to the existing markov models. Messages in the file must be the same format as ZNC
    writes them, [NN:NN:NN] <NICK> MSG. Usage: markov_loadfile <path> <channel_name>"""

    if len(argv) < 3:
        return

    try:
        dict_file_name = argv[1]
        with open(dict_file_name, errors='replace') as f:
            text = f.read()
    except:
        return "Couldn't read file"

    msg_regex = re.compile(r'^\[..:..:..\] <(.*?)> (.*)$')

    # go through all lines and extract nick + messages
    num_lines = 0
    for line in text.splitlines():
        match = msg_regex.match(line)
        if not match:
            continue
        nick = match.group(1)
        msg = match.group(2)
        spl = msg.split()
        if len(spl) < 3:
            continue
        m.add_sentence(nick, spl)
        m.add_sentence(argv[2], spl)
        num_lines += 1
    m.commit()
    return 'Loaded %d lines' % num_lines

def contains_mention(split):
    for w in split:
        if w.lower().strip(" ;:,.") == yui.get_nick().lower():
            return w
    return ''

# add new sentences as they come in
@yui.event('msgRecv')
def recv(channel, user, msg):
    # train the markov chains
    if channel == user.nick:  # ignore query
        return
    split = msg.split()
    m.add_sentence(user.nick, split)
    m.add_sentence(channel, split)
    m.commit()

    # do chatbot stuff
    if msg.startswith(tuple(yui.config_val('commandPrefixes', default=['!']))) or channel not in ACTIVE_CHANNELS:
        return

    mention = contains_mention(split)
    if mention != '' or random.random() < CHATTINESS:
        if mention != '':
            split.remove(mention)
            split.append(user.nick)
        topic = random.choice(split)
        sentence = generate(channel, [topic])
        if sentence != '':
            yui.send_msg(channel, yui.unhighlight_for_channel(sentence, channel))

@yui.command('markov', 'mark')
def markov(argv, user, channel):
    """Generate a random sentence for a given nick. Usage: markov [nick] [sentence_start]"""
    name = user.nick
    sentence_start = ['']
    if len(argv) > 1:
        name = argv[1]
    if len(argv) > 2:
        sentence_start = argv[2:2+STATE_SIZE]

    sentence = generate(name, sentence_start)
    if sentence == '':
        return "Couldn't generate a sentence :("
    return yui.unhighlight_for_channel('<%s> %s' % (name, sentence), channel)

@yui.command('chatbot')
def chatbot(channel, argv):
    """De-/activate random chatbot for a channel. Usage: chatbot [channel]"""
    global ACTIVE_CHANNELS
    if len(argv) > 1:
        channel = argv[1]

    if channel in ACTIVE_CHANNELS:
        ACTIVE_CHANNELS.remove(channel)
        return "I'll shut up"
    ACTIVE_CHANNELS.append(channel)
    return 'Chatting now'
<|endoftext|>


<commit_before><commit_msg>Fix api desktop creation timeout HTTP error code<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python3
from dhcp import *
from tkinter import *

root = Tk()
root.title('MAC, IP & Computer')
info_text = Text(root)
info_text.pack(fill = BOTH, expand = True)

configuration = DHCPServerConfiguration()
configuration.debug = print
#configuration.adjust_if_this_computer_is_a_router()
configuration.load('dhcpgui.conf')
server = DHCPServer(configuration)
server.run_in_thread()

last_time_sorted_entries = None
def update_text():
    global last_time_sorted_entries
    root.after(100, update_text)
    entries = server.get_current_entries()
    time_sorted_entries = list(reversed(sorted(entries, key = lambda entry: entry.last_received)))
    if last_time_sorted_entries != time_sorted_entries:
        text = 'MAC'.center(18) + '\t' + 'IP'.center(13) + '\t' + '  HOST' + '\n'
        headerlines = 1
        for entry in entries:
            text += '{}\t{}\t{}\n'.format(entry.mac, entry.ip, entry.hostname)
        last_text = text
        info_text.delete('0.0', END)
        info_text.insert(END, text)
        # prettify the text
        tags = []
        for time_i, entry in enumerate(time_sorted_entries):
            text_line = entries.index(entry) + 1 + headerlines
            start = "{}.0".format(text_line)
            stop  = "{}.0".format(text_line + 1)
            tag = 'time{}'.format(time_i)
            info_text.tag_add(tag, start, stop)
            tags.append(tag)
        for tag_i, tag in enumerate(tags):
            rank = 100 + int(tag_i * 155 / (len(tags)))
            hexrank = hex((rank >> 4) & 15)[-1] + hex(rank & 15)[-1]
            bg = '#ffff{}'.format(hexrank, hexrank)
            info_text.tag_config(tag, background = bg)
        last_time_sorted_entries = time_sorted_entries
        info_text.tag_raise("sel")

update_text()

root.mainloop()

server.close()
<commit_msg>added width and height<commit_after>#!/usr/bin/python3
from dhcp import *
from tkinter import *

root = Tk()
root.title('MAC, IP & Computer')
info_text = Text(root, height = 12, width = 45)
info_text.pack(fill = BOTH, expand = True)

configuration = DHCPServerConfiguration()
configuration.debug = print
#configuration.adjust_if_this_computer_is_a_router()
configuration.load('dhcpgui.conf')
server = DHCPServer(configuration)
server.run_in_thread()

last_time_sorted_entries = None
def update_text():
    global last_time_sorted_entries
    root.after(100, update_text)
    entries = server.get_current_entries()
    time_sorted_entries = list(reversed(sorted(entries, key = lambda entry: entry.last_received)))
    if last_time_sorted_entries != time_sorted_entries:
        text = 'MAC'.center(18) + '\t' + 'IP'.center(13) + '\t' + '  HOST' + '\n'
        headerlines = 1
        for entry in entries:
            text += '{}\t{}\t{}\n'.format(entry.mac, entry.ip, entry.hostname)
        last_text = text
        info_text.delete('0.0', END)
        info_text.insert(END, text)
        # prettify the text
        tags = []
        for time_i, entry in enumerate(time_sorted_entries):
            text_line = entries.index(entry) + 1 + headerlines
            start = "{}.0".format(text_line)
            stop  = "{}.0".format(text_line + 1)
            tag = 'time{}'.format(time_i)
            info_text.tag_add(tag, start, stop)
            tags.append(tag)
        for tag_i, tag in enumerate(tags):
            rank = 100 + int(tag_i * 155 / (len(tags)))
            hexrank = hex((rank >> 4) & 15)[-1] + hex(rank & 15)[-1]
            bg = '#ffff{}'.format(hexrank, hexrank)
            info_text.tag_config(tag, background = bg)
        last_time_sorted_entries = time_sorted_entries
        info_text.tag_raise("sel")

update_text()

root.mainloop()

server.close()
<|endoftext|>


<commit_before>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'banten'
		listkabkot = [
		'%3601%','%3602%','%3603%','%3604%',
		'%3671%','%3672%','%3673%','%3674%'
		]
		provloc = '106.143949, -6.460788'
		mapzoom = '9'
		kabkotcord = [
		'105.790328, -6.572485',
		'106.208272, -6.655552',
		'106.650031, -6.181121',
		'106.167182, -6.109267',
		'106.485223, -6.173674',
		'106.011024, -6.004235',
		'106.163997, -6.110311',
		'106.710786, -6.281848'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<commit_msg>36,73,BANTEN,SERANG<commit_after>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'banten'
		listkabkot = [
		'%3601%','%3602%','%3603%','%3604%',
		'%3671%','%3672%','%3673%','%3674%'
		]
		provloc = '106.143949, -6.460788'
		mapzoom = '9'
		kabkotcord = [
		'105.790328, -6.572485',
		'106.208272, -6.655552',
		'106.650031, -6.181121',
		'106.167182, -6.109267',
		'106.485223, -6.173674',
		'106.011024, -6.004235',
		'106.134223, -6.223955',
		'106.710786, -6.281848'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
hyper/http20/hpack_structures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Contains data structures used in hyper's HPACK implementation.
"""
class Reference(object):
    """
    The reference object is essentially an object that 'points to' another
    object, not unlike a pointer in C or similar languages. This object is
    distinct from the normal Python name because we can tell the difference
    between a reference and the 'actual' object.

    It behaves in the following ways:

    - Two references to the same object evaluate equal.
    - Two references to different objects evaluate not equal, even if those
      objects themselves evaluate equal.
    - Two references to the same object hash to the same value.
    - Two references to different objects hash to different values.

    The reference is distinct from and does not use weak references. A
    reference may never point at an object that has been garbage collected.
    This means that, to ensure objects get GC'd, any reference to them must
    also go out of scope.

    This object is _conceptually_ immutable, but the implementation doesn't
    attempt to enforce that to avoid the overhead. Be warned that changing
    the object being referenced after creation could lead to all sorts of weird
    nonsense.

    :param obj: The object being referenced.
    """
    # Enumerated states distinguishing between references that have been
    # emitted in this round of processing, those that haven't, and those that
    # have been 'implicitly' emitted (via the reference set).
    NOT_EMITTED = 0
    EMITTED = 1
    IMPLICITLY_EMITTED = 2

    def __init__(self, obj):
        self.obj = obj
        self.emitted = Reference.NOT_EMITTED

    def __eq__(self, other):
        return (isinstance(other, Reference) and self.obj is other.obj)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return id(self.obj)
<commit_msg>Add a repr for debugging.<commit_after># -*- coding: utf-8 -*-
"""
hyper/http20/hpack_structures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Contains data structures used in hyper's HPACK implementation.
"""
class Reference(object):
    """
    The reference object is essentially an object that 'points to' another
    object, not unlike a pointer in C or similar languages. This object is
    distinct from the normal Python name because we can tell the difference
    between a reference and the 'actual' object.

    It behaves in the following ways:

    - Two references to the same object evaluate equal.
    - Two references to different objects evaluate not equal, even if those
      objects themselves evaluate equal.
    - Two references to the same object hash to the same value.
    - Two references to different objects hash to different values.

    The reference is distinct from and does not use weak references. A
    reference may never point at an object that has been garbage collected.
    This means that, to ensure objects get GC'd, any reference to them must
    also go out of scope.

    This object is _conceptually_ immutable, but the implementation doesn't
    attempt to enforce that to avoid the overhead. Be warned that changing
    the object being referenced after creation could lead to all sorts of weird
    nonsense.

    :param obj: The object being referenced.
    """
    # Enumerated states distinguishing between references that have been
    # emitted in this round of processing, those that haven't, and those that
    # have been 'implicitly' emitted (via the reference set).
    NOT_EMITTED = 0
    EMITTED = 1
    IMPLICITLY_EMITTED = 2

    def __init__(self, obj):
        self.obj = obj
        self.emitted = Reference.NOT_EMITTED

    def __eq__(self, other):
        return (isinstance(other, Reference) and self.obj is other.obj)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return id(self.obj)

    def __repr__(self):  # pragma: no cover
        return 'Reference <%s>' % id(self.obj)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

def qrcode(message):
  userid = message.from_user.id
  m = bot.send_message(message.chat.id, "Please send the link/address/text/etc. you want to be converted to QR-Code")
  zigzag.nextstep(m, "qrnext")

def qrnext(message):
  userid = message.from_user.id
  args = message.text.replace("/qrcode ","").replace(" ", "%20")
  bot.reply_to(message, "Processing... Please wait !")
  urllib.urlretrieve("http://api.qrserver.com/v1/create-qr-code/?data={}&size=600x600".format(args), 'data/qr{}.png'.format(userid))
  bot.send_photo(message.chat.id, open('data/qr{}.png'.format(userid)), caption="QR Code by @TheZigZagBot")

class plqrcode:
  patterns = ["^[/!]qrcode(.*)$"]
<commit_msg>Update qrcode.py<commit_after># -*- coding: utf-8 -*-

def qrcode(message):
  userid = message.from_user.id
  m = bot.send_message(message.chat.id, "Please send the link/address/text/etc. you want to be converted to QR-Code")
  zigzag.nextstep(m, "qrnext")

def qrnext(message):
  userid = message.from_user.id
  args = message.text.replace("/qrcode ","").replace(" ", "%20")
  bot.reply_to(message, "Processing... Please wait !")
  urllib.urlretrieve("http://api.qrserver.com/v1/create-qr-code/?data={}&size=600x600".format(args), 'data/qr{}.png'.format(userid))
  bot.send_photo(message.chat.id, open('data/qr{}.png'.format(userid)), caption="QR Code by @TheZigZagBot")
  os.remove("data/qr{}.png".format(userid))

class plqrcode:
  patterns = ["^[/!]qrcode(.*)$"]
<|endoftext|>


<commit_before>import os.path
import sys

sys.path.append(os.path.join(os.path.dirname(__file__), "../"))

from unit_test_suite import UnitTestSuite

import copy
import re

class AlertTestSuite(UnitTestSuite):
    def setup(self):
        super(AlertTestSuite, self).setup()

        alerts_dir = os.path.join(os.path.dirname(__file__), "../../alerts/")
        os.chdir(alerts_dir)

        self.alert_classname = (self.__class__.__name__[4:] if
                                self.__class__.__name__.startswith('Test') else
                                False)

        if not self.alert_filename:
            # Convert "AlertFooBar" to "foo_bar" and "BazQux" to "baz_qux"
            self.alert_filename = re.sub(
                '([a-z0-9])([A-Z])',
                r'\1_\2',
                re.sub(
                    '(.)([A-Z][a-z]+)',
                    r'\1_\2',
                    self.alert_classname[5:] if
                    self.alert_classname.startswith('Alert') else
                    self.alert_classname)).lower()

    # Some housekeeping stuff here to make sure the data we get is 'good'
    def verify_starting_values(self, test_case):
        # Verify the description for the test case is populated
        assert test_case.description is not None or ""
        assert test_case.description is not ""

        # Verify alert_filename is a legit file
        # full_alert_file_path = "../../../alerts/" + self.alert_filename + ".py"
        full_alert_file_path = "./" + self.alert_filename + ".py"
        assert os.path.isfile(full_alert_file_path) is True

        # Verify we're able to load in the alert_classname
        # This can probably be improved, but for the mean time, we're just
        # gonna grep for class name
        alert_source_str = open(full_alert_file_path, 'r').read()
        class_search_str = "class " + self.alert_classname + "("
        assert class_search_str in alert_source_str

        # Verify events is not empty
        assert len(test_case.events) is not 0

        # Verify that if we're a positive test case, we actually passed in an
        # expected_alert
        if test_case.expected_test_result is True:
            assert test_case.expected_alert is not None
        else:
            # How can we expect an alert, when we're expecting the alert to
            # never throw?
            assert test_case.expected_alert is None

    # todo: remote this out to utilities
    def dict_merge(self, target, *args):
        # Merge multiple dicts
        if len(args) > 1:
            for obj in args:
                self.dict_merge(target, obj)
            return target

        # Recursively merge dicts and set non-dict values
        obj = args[0]
        if not isinstance(obj, dict):
            return obj
        for k, v in obj.iteritems():
            if k in target and isinstance(target[k], dict):
                self.dict_merge(target[k], v)
            else:
                target[k] = v
        return target

    def test_alert_test_case(self, test_case):
        self.verify_starting_values(test_case)
        temp_events = test_case.events
        for event in temp_events:
            temp_event = self.dict_merge(self.generate_default_event(), self.default_event)

            merged_event = self.dict_merge(temp_event, event)
            merged_event['_source']['utctimestamp'] = merged_event['_source']['utctimestamp']()
            test_case.full_events.append(merged_event)
            self.populate_test_event(
                merged_event['_source'], merged_event['_type'])

        self.es_client.flush('events')

        alert_task = test_case.run(alert_filename=self.alert_filename, alert_classname=self.alert_classname)
        self.verify_alert_task(alert_task, test_case)

    def verify_expected_alert(self, found_alert, test_case):
        # Verify index is set correctly
        assert found_alert['_index'] == self.alert_index_name
        # Verify alert type is correct
        assert found_alert['_type'] == 'alert'

        # Verify that the alert has the right "look to it"
        assert found_alert.keys() == [
            '_score', '_type', '_id', '_source', '_index']

        # Verify the alert has an id field that is unicode
        assert type(found_alert['_id']) == unicode

        # Verify there is a utctimestamp field
        assert 'utctimestamp' in found_alert['_source']

        # Verify the events are added onto the alert
        assert type(found_alert['_source']['events']) == list
        assert len(found_alert['_source']['events']) is not 0
        alert_events = found_alert['_source']['events']
        sorted_alert_events = sorted(alert_events, key=lambda k: k[
                                     'documentsource']['utctimestamp'])

        created_events = test_case.full_events
        sorted_created_events = sorted(created_events, key=lambda k: k[
                                       '_source']['utctimestamp'])

        event_index = 0
        for event in sorted_alert_events:
            assert event['documentsource'] == sorted_created_events[
                event_index]['_source']
            event_index += 1

        # Verify that the alert properties are set correctly
        for key, value in test_case.expected_alert.iteritems():
            assert found_alert['_source'][key] == value

    def verify_alert_task(self, alert_task, test_case):
        if test_case.expected_test_result is True:
            assert len(alert_task.alert_ids) is not 0
            self.es_client.flush('alerts')
            for alert_id in alert_task.alert_ids:
                found_alert = self.es_client.get_alert_by_id(alert_id)
                self.verify_expected_alert(found_alert, test_case)
        else:
            assert len(alert_task.alert_ids) is 0

    @staticmethod
    def copy(obj):
        return copy.deepcopy(obj)
<commit_msg>Add ability to customize alert classname<commit_after>import os.path
import sys

sys.path.append(os.path.join(os.path.dirname(__file__), "../"))

from unit_test_suite import UnitTestSuite

import copy
import re

class AlertTestSuite(UnitTestSuite):
    def setup(self):
        super(AlertTestSuite, self).setup()

        alerts_dir = os.path.join(os.path.dirname(__file__), "../../alerts/")
        os.chdir(alerts_dir)

        if not self.alert_classname:
            self.alert_classname = (self.__class__.__name__[4:] if
                                    self.__class__.__name__.startswith('Test') else
                                    False)

        if not self.alert_filename:
            # Convert "AlertFooBar" to "foo_bar" and "BazQux" to "baz_qux"
            self.alert_filename = re.sub(
                '([a-z0-9])([A-Z])',
                r'\1_\2',
                re.sub(
                    '(.)([A-Z][a-z]+)',
                    r'\1_\2',
                    self.alert_classname[5:] if
                    self.alert_classname.startswith('Alert') else
                    self.alert_classname)).lower()

    # Some housekeeping stuff here to make sure the data we get is 'good'
    def verify_starting_values(self, test_case):
        # Verify the description for the test case is populated
        assert test_case.description is not None or ""
        assert test_case.description is not ""

        # Verify alert_filename is a legit file
        # full_alert_file_path = "../../../alerts/" + self.alert_filename + ".py"
        full_alert_file_path = "./" + self.alert_filename + ".py"
        assert os.path.isfile(full_alert_file_path) is True

        # Verify we're able to load in the alert_classname
        # This can probably be improved, but for the mean time, we're just
        # gonna grep for class name
        alert_source_str = open(full_alert_file_path, 'r').read()
        class_search_str = "class " + self.alert_classname + "("
        assert class_search_str in alert_source_str

        # Verify events is not empty
        assert len(test_case.events) is not 0

        # Verify that if we're a positive test case, we actually passed in an
        # expected_alert
        if test_case.expected_test_result is True:
            assert test_case.expected_alert is not None
        else:
            # How can we expect an alert, when we're expecting the alert to
            # never throw?
            assert test_case.expected_alert is None

    # todo: remote this out to utilities
    def dict_merge(self, target, *args):
        # Merge multiple dicts
        if len(args) > 1:
            for obj in args:
                self.dict_merge(target, obj)
            return target

        # Recursively merge dicts and set non-dict values
        obj = args[0]
        if not isinstance(obj, dict):
            return obj
        for k, v in obj.iteritems():
            if k in target and isinstance(target[k], dict):
                self.dict_merge(target[k], v)
            else:
                target[k] = v
        return target

    def test_alert_test_case(self, test_case):
        self.verify_starting_values(test_case)
        temp_events = test_case.events
        for event in temp_events:
            temp_event = self.dict_merge(self.generate_default_event(), self.default_event)

            merged_event = self.dict_merge(temp_event, event)
            merged_event['_source']['utctimestamp'] = merged_event['_source']['utctimestamp']()
            test_case.full_events.append(merged_event)
            self.populate_test_event(
                merged_event['_source'], merged_event['_type'])

        self.es_client.flush('events')

        alert_task = test_case.run(alert_filename=self.alert_filename, alert_classname=self.alert_classname)
        self.verify_alert_task(alert_task, test_case)

    def verify_expected_alert(self, found_alert, test_case):
        # Verify index is set correctly
        assert found_alert['_index'] == self.alert_index_name
        # Verify alert type is correct
        assert found_alert['_type'] == 'alert'

        # Verify that the alert has the right "look to it"
        assert found_alert.keys() == [
            '_score', '_type', '_id', '_source', '_index']

        # Verify the alert has an id field that is unicode
        assert type(found_alert['_id']) == unicode

        # Verify there is a utctimestamp field
        assert 'utctimestamp' in found_alert['_source']

        # Verify the events are added onto the alert
        assert type(found_alert['_source']['events']) == list
        assert len(found_alert['_source']['events']) is not 0
        alert_events = found_alert['_source']['events']
        sorted_alert_events = sorted(alert_events, key=lambda k: k[
                                     'documentsource']['utctimestamp'])

        created_events = test_case.full_events
        sorted_created_events = sorted(created_events, key=lambda k: k[
                                       '_source']['utctimestamp'])

        event_index = 0
        for event in sorted_alert_events:
            assert event['documentsource'] == sorted_created_events[
                event_index]['_source']
            event_index += 1

        # Verify that the alert properties are set correctly
        for key, value in test_case.expected_alert.iteritems():
            assert found_alert['_source'][key] == value

    def verify_alert_task(self, alert_task, test_case):
        if test_case.expected_test_result is True:
            assert len(alert_task.alert_ids) is not 0
            self.es_client.flush('alerts')
            for alert_id in alert_task.alert_ids:
                found_alert = self.es_client.get_alert_by_id(alert_id)
                self.verify_expected_alert(found_alert, test_case)
        else:
            assert len(alert_task.alert_ids) is 0

    @staticmethod
    def copy(obj):
        return copy.deepcopy(obj)
<|endoftext|>


<commit_before>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'sulsel'
		provloc = '119.9740534, -3.6687994'
		mapzoom = '9'
		kabkotcord = [
		'120.481566, -6.040677',
		'120.214471, -5.415704',
		'120.026454, -5.513190',
		'119.970836, -5.461244',
		'119.496695, -5.408397',
		'18.119.6730939,-5.554579',
	 	'19.119.6730939,-5.554579',
	  	'20.119.4875668,-5.4162493',
	  	'21.19.742604,-5.3102888',
	  	'22.120.112735,-5.2171961',
	  	'23.119.6962677,-5.0549145',
	  	'24.119.5571677,-4.805035',
	  	'25.119.6499162,-4.436417',
	  	'26.120.0665236,-4.7443383',
	  	#'120.066524, -4.022229',
		#'120.020296, -3.773898',
                #'119.557168, -3.648349',
		#'119.88152, -3.459074',
		#'120.251273, -3.305221 ',
		#'119.742604, -3.0753',
		'119.974053, -2.269045',
		'121.171039, -2.582552',
		'119.83523, -2.862194',
		'119.432731,-5.147665'
		]
		listkabkot = [
		'%7301%','%7302%','%7303%','%7304%','%7305%','%7306%','%7307%','%7308%','%7309%','%7310%',
		'%7311%','%7312%','%7313%','%7314%','%7315%','%7316%','%7317%','%7318%',
		'%7322%','%7325%','%7326%',
		'%7371%','%7372%','%7373%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt
<commit_msg>tambah tana toraja<commit_after>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'sulsel'
		provloc = '119.9740534, -3.6687994'
		mapzoom = '9'
		kabkotcord = [
		'120.481566, -6.040677',
		'120.214471, -5.415704',
		'120.026454, -5.513190',
		'119.970836, -5.461244',
		'119.496695, -5.408397',
		'18.119.6730939,-5.554579',
	 	'19.119.6730939,-5.554579',
	  	'20.119.4875668,-5.4162493',
	  	'21.19.742604,-5.3102888',
	  	'22.120.112735,-5.2171961',
	  	'23.119.6962677,-5.0549145',
	  	'24.119.5571677,-4.805035',
	  	'25.119.6499162,-4.436417',
	  	'26.120.0665236,-4.7443383',
	  	#'120.066524, -4.022229',
		#'120.020296, -3.773898',
                #'119.557168, -3.648349',
		#'119.88152, -3.459074',
		#'120.251273, -3.305221 ',
		'119.742604, -3.0753',
		'119.974053, -2.269045',
		'121.171039, -2.582552',
		'119.83523, -2.862194',
		'119.432731,-5.147665'
		]
		listkabkot = [
		'%7301%','%7302%','%7303%','%7304%','%7305%','%7306%','%7307%','%7308%','%7309%','%7310%',
		'%7311%','%7312%','%7313%','%7314%','%7315%','%7316%','%7317%','%7318%',
		'%7322%','%7325%','%7326%',
		'%7371%','%7372%','%7373%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt
<|endoftext|>


<commit_before>
from hypothesis import given
from hypothesis.strategies import integers

from numba import jit
import numpy as np

@jit(nopython=True, parallel=True)
def get(n):
    return np.ones((n,1), dtype=np.float64)

@given(integers(min_value=10, max_value=100000))
def test_all_ones(x):
    """
    We found one of the scaling tests failing on
    OS X with numba 0.35, but it passed on other
    platforms, and passed consistently with numba
    0.36.

    The issue appears to be the same as numba#2609
    https://github.com/numba/numba/issues/2609
    so we've taken the minimal repro from that issue
    and are using it as a unit-test here.
    """
    result = get(x)
    expected = np.ones((x, 1), dtype=np.float64)
    assert np.allclose(expected, result)

if __name__ == '__main__':
    import pytest
    pytest.main([__file__])
<commit_msg>Add guard for parallel kwarg on 32-bit Windows<commit_after>
import sys

from hypothesis import given
from hypothesis.strategies import integers

from numba import jit
import numpy as np

# Parallel not supported on 32-bit Windows
parallel = not (sys.platform == 'win32')

@jit(nopython=True, parallel=True)
def get(n):
    return np.ones((n,1), dtype=np.float64)

@given(integers(min_value=10, max_value=100000))
def test_all_ones(x):
    """
    We found one of the scaling tests failing on
    OS X with numba 0.35, but it passed on other
    platforms, and passed consistently with numba
    0.36.

    The issue appears to be the same as numba#2609
    https://github.com/numba/numba/issues/2609
    so we've taken the minimal repro from that issue
    and are using it as a unit-test here.
    """
    result = get(x)
    expected = np.ones((x, 1), dtype=np.float64)
    assert np.allclose(expected, result)

if __name__ == '__main__':
    import pytest
    pytest.main([__file__])
<|endoftext|>