<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from __future__ import print_function

import cv2
import sys
import time
import traceback

from ikalog.utils import *
from . import scenes

# The IkaLog core engine.
#

class IkaEngine:

    #    def on_game_start(self, context):
    #        self.scn_tower_tracker.reset(context)

    def on_game_individual_result(self, context):
        self.session_close_wdt = context['engine']['msec'] + (20 * 1000)

    def on_result_gears(self, context):
        if self.session_close_wdt is not None:
            self.session_close_wdt = context['engine']['msec'] + (1 * 1000)

    def dprint(self, text):
        print(text, file=sys.stderr)

    def call_plugins(self, event_name, debug=False):
        if debug:
            self.dprint('call plug-in hook (%s):' % event_name)

        for op in self.output_plugins:
            if hasattr(op, event_name):
                if debug:
                    self.dprint('Call  %s' % op.__class__.__name__)
                try:
                    getattr(op, event_name)(self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')
            elif hasattr(op, 'onUncatchedEvent'):
                if debug:
                    self.dprint(
                        'call plug-in hook (UncatchedEvent, %s):' % event_name)
                try:
                    getattr(op, 'onUncatchedEvent')(event_name, self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')

    def call_plugins_later(self, event_name, debug=False):
        self._event_queue.append(event_name)

    def read_next_frame(self, skip_frames=0):
        for i in range(skip_frames):
            frame = self.capture.read_frame()
        frame = self.capture.read_frame()

        while frame is None:
            self.call_plugins('on_frame_read_failed')
            if self._stop:
                return None, None
            cv2.waitKey(1000)
            frame = self.capture.read_frame()

        t = self.capture.get_current_timestamp()
        self.context['engine']['msec'] = t
        self.context['engine']['frame'] = frame

        self.call_plugins('on_debug_read_next_frame')

        return frame, t

    def stop(self):
        self.call_plugins('on_stop')
        self._stop = True

    def reset(self):
        # Initalize the context
        self.context['game'] = {
            'map': None,
            'rule': None,
            'won': None,
            'players': None,

            'kills': 0,
            'dead': False,
            'death_reasons': {},

            'inkling_state': [None, None],
            'livesTrack': [],
            'towerTrack': [],
        }
        self.call_plugins('on_game_reset')

    def create_context(self):
        self.context = {
            'engine': {
                'frame': None,
                'service': {
                    'callPlugins': self.call_plugins,
                }
            },
            'scenes': {
            },
            'config': {
            },
            'lobby': {
            }
        }
        self.reset()
        self.session_close_wdt = None

    def session_close(self):
        self.session_close_wdt = None

        self.call_plugins('on_game_session_end')
        self.reset()

    def process_scene(self, scene):
        context = self.context

        scene.new_frame(context)
        r = scene.match(context)
        if r and scene.exclusive_scene:
            print('%s: entering %s' % (self, scene.__class__.__name__))
            while r and (not self._stop):
                frame, t = self.read_next_frame()
                if frame is None:
                    continue
                context['engine']['frame'] = frame
                scene.new_frame(context)
                r = scene.match(context)
                if r:
                    scene.analyze(context)
            print('%s: escaping %s' % (self, scene.__class__.__name__))

    def find_scene_object(self, scene_class_name):
        for scene in self.scenes:
            if scene.__class__.__name__ == scene_class_name:
                return scene
        return None

    def process_frame(self):
        context = self.context

        frame, t = self.read_next_frame()

        if frame is None:
            return False

        context['engine']['inGame'] = self.find_scene_object(
            'GameTimerIcon').match(context)

        self.call_plugins('on_frame_read')

        for scene in self.scenes:
            self.process_scene(scene)

        if self.session_close_wdt is not None:
            if self.session_close_wdt < context['engine']['msec']:
                self.dprint('Watchdog fired. Closing current session')
                self.session_close()

        key = None

        # FixMe: Since on_frame_next and on_key_press has non-standard arguments,
        # self.call_plugins() doesn't work for those.

        for op in self.output_plugins:
            if hasattr(op, "on_frame_next"):
                try:
                    key = op.on_frame_next(context)
                except:
                    pass

        for op in self.output_plugins:
            if hasattr(op, "on_key_press"):
                try:
                    op.on_key_press(context, key)
                except:
                    pass

        while len(self._event_queue) > 0:
            self.call_plugins(self._event_queue.pop(0))

    def run(self):
        # Main loop.
        while not self._stop:
            if self._pause:
                time.sleep(0.5)
                continue

            try:
                self.process_frame()
            except EOFError:
                # EOF. Close session if close_session_at_eof is set.
                if self.close_session_at_eof:
                    if self.session_close_wdt is not None:
                        self.dprint('Closing current session at EOF')
                        self.session_close()
                raise

        cv2.destroyAllWindows()

    def set_capture(self, capture):
        self.capture = capture

    def set_plugins(self, plugins):
        self.output_plugins = [self]
        self.output_plugins.extend(self.scenes)
        self.output_plugins.extend(plugins)

    def pause(self, pause):
        self._pause = pause

    def _initialize_scenes(self):
        self.scenes = [
            scenes.GameTimerIcon(self),
            scenes.GameStart(self),
            scenes.GameGoSign(self),
            scenes.GameKill(self),
            scenes.GameDead(self),
            scenes.GameOutOfBound(self),
            scenes.GameFinish(self),
            scenes.GameSpecialGauge(self),
            scenes.GameSpecialWeapon(self),
            scenes.ResultJudge(self),

            scenes.GameRankedBattleEvents(self),
            scenes.PaintScoreTracker(self),
            scenes.ObjectiveTracker(self),
            scenes.SplatzoneTracker(self),
            scenes.InklingsTracker(self),

            scenes.ResultDetail(self),
            scenes.ResultUdemae(self),
            scenes.ResultGears(self),
            scenes.ResultFesta(self),
            scenes.Lobby(self),
        ]

    def __init__(self):
        self._initialize_scenes()

        self.output_plugins = [self]
        self.last_capture = time.time() - 100

        self._stop = False
        self._pause = True
        self._event_queue = []

        self.close_session_at_eof = False

        self.create_context()
<commit_msg>ikalog/engine: Shutdown IkaLog engine gracefully when reached at the end of file.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from __future__ import print_function

import cv2
import sys
import time
import traceback

from ikalog.utils import *
from . import scenes

# The IkaLog core engine.
#

class IkaEngine:

    #    def on_game_start(self, context):
    #        self.scn_tower_tracker.reset(context)

    def on_game_individual_result(self, context):
        self.session_close_wdt = context['engine']['msec'] + (20 * 1000)

    def on_result_gears(self, context):
        if self.session_close_wdt is not None:
            self.session_close_wdt = context['engine']['msec'] + (1 * 1000)

    def dprint(self, text):
        print(text, file=sys.stderr)

    def call_plugins(self, event_name, debug=False):
        if debug:
            self.dprint('call plug-in hook (%s):' % event_name)

        for op in self.output_plugins:
            if hasattr(op, event_name):
                if debug:
                    self.dprint('Call  %s' % op.__class__.__name__)
                try:
                    getattr(op, event_name)(self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')
            elif hasattr(op, 'onUncatchedEvent'):
                if debug:
                    self.dprint(
                        'call plug-in hook (UncatchedEvent, %s):' % event_name)
                try:
                    getattr(op, 'onUncatchedEvent')(event_name, self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')

    def call_plugins_later(self, event_name, debug=False):
        self._event_queue.append(event_name)

    def read_next_frame(self, skip_frames=0):
        for i in range(skip_frames):
            frame = self.capture.read_frame()
        frame = self.capture.read_frame()

        while frame is None:
            self.call_plugins('on_frame_read_failed')
            if self._stop:
                return None, None
            cv2.waitKey(1000)
            frame = self.capture.read_frame()

        t = self.capture.get_current_timestamp()
        self.context['engine']['msec'] = t
        self.context['engine']['frame'] = frame

        self.call_plugins('on_debug_read_next_frame')

        return frame, t

    def stop(self):
        self.call_plugins('on_stop')
        self._stop = True

    def reset(self):
        # Initalize the context
        self.context['game'] = {
            'map': None,
            'rule': None,
            'won': None,
            'players': None,

            'kills': 0,
            'dead': False,
            'death_reasons': {},

            'inkling_state': [None, None],
            'livesTrack': [],
            'towerTrack': [],
        }
        self.call_plugins('on_game_reset')

    def create_context(self):
        self.context = {
            'engine': {
                'frame': None,
                'service': {
                    'callPlugins': self.call_plugins,
                }
            },
            'scenes': {
            },
            'config': {
            },
            'lobby': {
            }
        }
        self.reset()
        self.session_close_wdt = None

    def session_close(self):
        self.session_close_wdt = None

        self.call_plugins('on_game_session_end')
        self.reset()

    def process_scene(self, scene):
        context = self.context

        scene.new_frame(context)
        r = scene.match(context)
        if r and scene.exclusive_scene:
            print('%s: entering %s' % (self, scene.__class__.__name__))
            while r and (not self._stop):
                frame, t = self.read_next_frame()
                if frame is None:
                    continue
                context['engine']['frame'] = frame
                scene.new_frame(context)
                r = scene.match(context)
                if r:
                    scene.analyze(context)
            print('%s: escaping %s' % (self, scene.__class__.__name__))

    def find_scene_object(self, scene_class_name):
        for scene in self.scenes:
            if scene.__class__.__name__ == scene_class_name:
                return scene
        return None

    def process_frame(self):
        context = self.context

        frame, t = self.read_next_frame()

        if frame is None:
            return False

        context['engine']['inGame'] = self.find_scene_object(
            'GameTimerIcon').match(context)

        self.call_plugins('on_frame_read')

        for scene in self.scenes:
            self.process_scene(scene)

        if self.session_close_wdt is not None:
            if self.session_close_wdt < context['engine']['msec']:
                self.dprint('Watchdog fired. Closing current session')
                self.session_close()

        key = None

        # FixMe: Since on_frame_next and on_key_press has non-standard arguments,
        # self.call_plugins() doesn't work for those.

        for op in self.output_plugins:
            if hasattr(op, "on_frame_next"):
                try:
                    key = op.on_frame_next(context)
                except:
                    pass

        for op in self.output_plugins:
            if hasattr(op, "on_key_press"):
                try:
                    op.on_key_press(context, key)
                except:
                    pass

        while len(self._event_queue) > 0:
            self.call_plugins(self._event_queue.pop(0))

    def run(self):
        # Main loop.
        while not self._stop:
            if self._pause:
                time.sleep(0.5)
                continue

            try:
                self.process_frame()
            except EOFError:
                # EOF. Close session if close_session_at_eof is set.
                if self.close_session_at_eof:
                    if self.session_close_wdt is not None:
                        self.dprint('Closing current session at EOF')
                        self.session_close()
                self._stop = True

        cv2.destroyAllWindows()

    def set_capture(self, capture):
        self.capture = capture

    def set_plugins(self, plugins):
        self.output_plugins = [self]
        self.output_plugins.extend(self.scenes)
        self.output_plugins.extend(plugins)

    def pause(self, pause):
        self._pause = pause

    def _initialize_scenes(self):
        self.scenes = [
            scenes.GameTimerIcon(self),
            scenes.GameStart(self),
            scenes.GameGoSign(self),
            scenes.GameKill(self),
            scenes.GameDead(self),
            scenes.GameOutOfBound(self),
            scenes.GameFinish(self),
            scenes.GameSpecialGauge(self),
            scenes.GameSpecialWeapon(self),
            scenes.ResultJudge(self),

            scenes.GameRankedBattleEvents(self),
            scenes.PaintScoreTracker(self),
            scenes.ObjectiveTracker(self),
            scenes.SplatzoneTracker(self),
            scenes.InklingsTracker(self),

            scenes.ResultDetail(self),
            scenes.ResultUdemae(self),
            scenes.ResultGears(self),
            scenes.ResultFesta(self),
            scenes.Lobby(self),
        ]

    def __init__(self):
        self._initialize_scenes()

        self.output_plugins = [self]
        self.last_capture = time.time() - 100

        self._stop = False
        self._pause = True
        self._event_queue = []

        self.close_session_at_eof = False

        self.create_context()
<|endoftext|>


<commit_before>from django.conf import settings
from appconf import AppConf

import tempfile
import os

class ImagefitConf(AppConf):

    #: dictionary of preset names that have width and height values
    IMAGEFIT_PRESETS = {
        'thumbnail': {'width': 80, 'height': 80, 'crop': True},
        'medium': {'width': 320, 'height': 240},
        'original': {},
    }

    #: dictionary of output formats depending on the output image extension.
    IMAGEFIT_EXT_TO_FORMAT = {
        '.jpg': 'jpeg', '.jpeg': 'jpeg'
    }

    #: default format for any missing extension in IMAGEFIT_EXT_TO_FORMAT
    #: do not fall-back to a default format but raise an exception if set to None
    IMAGEFIT_EXT_TO_FORMAT_DEFAULT = 'png'

    #: root path from where to read urls
    IMAGEFIT_ROOT = ''

    IMAGEFIT_CACHE_ENABLED = True
    IMAGEFIT_CACHE_BACKEND_NAME = 'imagefit'

    settings.CACHES[IMAGEFIT_CACHE_BACKEND_NAME] = {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': os.path.join(tempfile.gettempdir(), 'django_imagefit')
    }

    #: ConditionalGetMiddleware is required for browser caching
    # Django's middleware management (and some other things) has changed
    # since version 1.9.
    property_name = (hasattr(settings, 'MIDDLEWARE') and 'MIDDLEWARE') or 'MIDDLEWARE_CLASSES'
    middleswares = getattr(settings, property_name)
    if not 'django.middleware.http.ConditionalGetMiddleware' in property_name:
        setattr(settings, property_name, middlewares + ('django.middleware.http.ConditionalGetMiddleware',))

def ext_to_format(filename):
    extension = os.path.splitext(filename)[1].lower()
    format = settings.IMAGEFIT_EXT_TO_FORMAT.get(extension, settings.IMAGEFIT_EXT_TO_FORMAT_DEFAULT)
    if not format:
        raise KeyError('Unknown image extension: {0}'.format(extension))
    return format
<commit_msg>Changes 'not X in Y' to 'X not in Y'<commit_after>from django.conf import settings
from appconf import AppConf

import tempfile
import os

class ImagefitConf(AppConf):

    #: dictionary of preset names that have width and height values
    IMAGEFIT_PRESETS = {
        'thumbnail': {'width': 80, 'height': 80, 'crop': True},
        'medium': {'width': 320, 'height': 240},
        'original': {},
    }

    #: dictionary of output formats depending on the output image extension.
    IMAGEFIT_EXT_TO_FORMAT = {
        '.jpg': 'jpeg', '.jpeg': 'jpeg'
    }

    #: default format for any missing extension in IMAGEFIT_EXT_TO_FORMAT
    #: do not fall-back to a default format but raise an exception if set to None
    IMAGEFIT_EXT_TO_FORMAT_DEFAULT = 'png'

    #: root path from where to read urls
    IMAGEFIT_ROOT = ''

    IMAGEFIT_CACHE_ENABLED = True
    IMAGEFIT_CACHE_BACKEND_NAME = 'imagefit'

    settings.CACHES[IMAGEFIT_CACHE_BACKEND_NAME] = {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': os.path.join(tempfile.gettempdir(), 'django_imagefit')
    }

    #: ConditionalGetMiddleware is required for browser caching
    # Django's middleware management (and some other things) has changed
    # since version 1.9.
    property_name = (hasattr(settings, 'MIDDLEWARE') and 'MIDDLEWARE') or 'MIDDLEWARE_CLASSES'
    middleswares = getattr(settings, property_name)
    if 'django.middleware.http.ConditionalGetMiddleware' not in property_name:
        setattr(settings, property_name, middlewares + ('django.middleware.http.ConditionalGetMiddleware',))

def ext_to_format(filename):
    extension = os.path.splitext(filename)[1].lower()
    format = settings.IMAGEFIT_EXT_TO_FORMAT.get(extension, settings.IMAGEFIT_EXT_TO_FORMAT_DEFAULT)
    if not format:
        raise KeyError('Unknown image extension: {0}'.format(extension))
    return format
<|endoftext|>


<commit_before># encoding: utf-8

from functools import partial
from marrow.package.loader import load
from marrow.package.host import PluginManager
from mongoengine import ImageGridFsProxy
from web.contentment.util import D_

from web.component.asset.model import Asset

log = __import__('logging').getLogger(__name__)

def indent(context, lines, padding='\t'):
	return padding + ('\n' + padding).join(lines.split('\n'))

MAP = {
		'localhost': ('career.nse-automatech.com', 'fr', 'http://localhost:8080/'),

		# NSE Automatech
		'app-15b75793-4441-4fe2-9c12-1f13b90d00be.cleverapps.io': ('career.nse-automatech.com', 'en', 'http://app-15b75793-4441-4fe2-9c12-1f13b90d00be.cleverapps.io'),
		# Testing URLs
		'en.nse.illico.cleverapps.io': ('career.nse-automatech.com', 'en', 'fr.nse.illico.cleverapps.io'),
		'fr.nse.illico.cleverapps.io': ('career.nse-automatech.com', 'fr', 'en.nse.illico.cleverapps.io'),
		# Production URLs
		'career.nse-automatech.com': ('career.nse-automatech.com', 'en', 'http://carrieres.nse-automatech.com'),
		'carrieres.nse-automatech.com': ('career.nse-automatech.com', 'fr', 'http://career.nse-automatech.com'),
	}

class ContentmentExtension:
	needs = ()

	def start(self, context):
		log = __import__('logging').getLogger(__name__)
		log.info("Starting Contentment extension.")

		for asset_type in PluginManager('web.component'):
			log.info("Found asset type: " + repr(asset_type))

		context.view.register(dict, self.render_json_response)
		context.view.register(ImageGridFsProxy, self.render_gridfs_image)

	def prepare(self, context):
		dom = context.request.host.partition(':')[0]
		parts = MAP.get(dom, (dom, 'en'))
		context.domain = parts[0]
		context.lang = parts[1]
		context.otherlang = parts[2]
		context.croot = Asset.objects.nearest('/' + context.domain)
		context.D = partial(D_, lang=parts[1])
		context.replacements = dict(context=context)

		if context.croot:
			context.theme = load(context.croot.properties.theme + ':page')
		else:
			context.theme = load('web.theme.bootstrap.base:page')

		log.info("Prepared context.", extra=dict(domain=[dom, context.domain], lang=context.lang, root=repr(context.croot), theme=repr(context.theme)))

	def render_json_response(self, context, result):
		import json

		response = context.response

		response.content_type = 'application/json'
		response.encoding = 'utf-8'
		response.text = json.dumps(result)

		return True

	def render_gridfs_image(self, context, result):
		response = context.response

		response.content_type = 'image/' + result.format.lower()
		result.seek(0)
		response.body = result.read()

		return True
<commit_msg>Context tweaks.<commit_after># encoding: utf-8

from functools import partial
from marrow.package.loader import load
from marrow.package.host import PluginManager
from mongoengine import ImageGridFsProxy
from web.contentment.util import D_

from web.component.asset.model import Asset

log = __import__('logging').getLogger(__name__)

def indent(context, lines, padding='\t'):
	return padding + ('\n' + padding).join(lines.split('\n'))

MAP = {
		'localhost': ('localhost', 'en', 'http://localhost:8080/'),
	}

class ContentmentExtension:
	needs = ()

	def start(self, context):
		log = __import__('logging').getLogger(__name__)
		log.info("Starting Contentment extension.")

		for asset_type in PluginManager('web.component'):
			log.info("Found asset type: " + repr(asset_type))

		context.view.register(dict, self.render_json_response)
		context.view.register(ImageGridFsProxy, self.render_gridfs_image)

	def prepare(self, context):
		dom = context.request.host.partition(':')[0]
		parts = MAP.get(dom, (dom, 'en', ''))
		context.domain = parts[0]
		context.lang = parts[1]
		context.otherlang = parts[2]
		context.site = context.croot = Asset.objects.nearest('/' + context.domain)
		context.D = partial(D_, lang=parts[1])
		context.replacements = dict(context=context)

		if context.croot:
			context.theme = load(context.croot.properties.theme + ':page')
		else:
			context.theme = load('web.theme.bootstrap.base:page')

		log.info("Prepared context.", extra=dict(domain=[dom, context.domain], lang=context.lang, root=repr(context.croot), theme=repr(context.theme)))

	def render_json_response(self, context, result):
		import json

		response = context.response

		response.content_type = 'application/json'
		response.encoding = 'utf-8'
		response.text = json.dumps(result)

		return True

	def render_gridfs_image(self, context, result):
		response = context.response

		response.content_type = 'image/' + result.format.lower()
		result.seek(0)
		response.body = result.read()

		return True
<|endoftext|>


<commit_before>from django.contrib import admin

from .models import UserThread, Message

class UserThreadAdmin(admin.ModelAdmin):
    list_display = ["thread", "user", "unread", "deleted"]
    list_filter = ["unread", "deleted"]
    raw_id_fields = ["user"]

class MessageAdmin(admin.ModelAdmin):
    list_display = ["thread", "sender", "sent_at"]
    list_filter = ["sent_at", "thread"]
    raw_id_fields = ["sender"]

admin.site.register[Message, MessageAdmin]
admin.site.register[UserThread, UserThreadAdmin]
<commit_msg>Fix typo<commit_after>from django.contrib import admin

from .models import UserThread, Message

class UserThreadAdmin(admin.ModelAdmin):
    list_display = ["thread", "user", "unread", "deleted"]
    list_filter = ["unread", "deleted"]
    raw_id_fields = ["user"]

class MessageAdmin(admin.ModelAdmin):
    list_display = ["thread", "sender", "sent_at"]
    list_filter = ["sent_at", "thread"]
    raw_id_fields = ["sender"]

admin.site.register(Message, MessageAdmin)
admin.site.register(UserThread, UserThreadAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2018, Foundries.io Ltd
# Copyright (c) 2018, Nordic Semiconductor ASA
# Copyright (c) 2017, Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

# Internal script used by the documentation's build system to create
# the "final" docs tree which is then compiled by Sphinx.
#
# This works around the fact that Sphinx needs a single documentation
# root directory, while Zephyr's documentation files are spread around
# the tree.

import argparse
import collections
import fnmatch
import os
from os import path
import re
import shutil
import sys

# directives to parse for included files
DIRECTIVES = ["figure", "include", "image", "literalinclude"]

# A simple namedtuple for a generated output file.
#
# - src: source file, what file should be copied (in source directory)
# - dst: destination file, path it should be copied to (in build directory)
Output = collections.namedtuple('Output', 'src dst')

# Represents the content which must be extracted from the Zephyr tree,
# as well as the output directories needed to contain it.
#
# - outputs: list of Output objects for extracted content.
# - output_dirs: set of directories which must exist to contain
#                output destination files.
Content = collections.namedtuple('Content', 'outputs output_dirs')

def src_deps(zephyr_base, src_file, dest, src_root):
    # - zephyr_base: the ZEPHYR_BASE directory containing src_file
    # - src_file: path to a source file in the documentation
    # - dest: path to the top-level output/destination directory
    # - src_root: path to the Sphinx top-level source directory
    #
    # Return a list of Output objects which contain src_file's
    # additional dependencies, as they should be copied into
    # dest. Output paths inside dest are based on each
    # dependency's relative path from zephyr_base.

    # Inspect only .rst files for directives referencing other files
    # we'll need to copy (as configured in the DIRECTIVES variable)
    if not src_file.endswith(".rst"):
        return []

    # Load the file's contents, bailing on decode errors.
    try:
        with open(src_file, encoding="utf-8") as f:
            content = [x.strip() for x in f.readlines()]
    except UnicodeDecodeError as e:
        sys.stderr.write(
            "Malformed {} in {}\n"
            "  Context: {}\n"
            "  Problematic data: {}\n"
            "  Reason: {}\n".format(
                e.encoding, src_file,
                e.object[max(e.start - 40, 0):e.end + 40],
                e.object[e.start:e.end],
                e.reason))
        return []

    # Source file's directory.
    src_dir = path.dirname(src_file)
    # Destination directory for any dependencies.
    dst_dir = path.join(dest, path.relpath(src_dir, start=zephyr_base))

    # Find directives in the content which imply additional
    # dependencies. We assume each such directive takes a single
    # argument, which is a (relative) path to the additional
    # dependency file.
    directives = "|".join(DIRECTIVES)
    pattern = re.compile(r"\.\.\s+(?P<directive>%s)::\s+(?P<dep_rel>.*)" %
                         directives)
    deps = []
    for l in content:
        m = pattern.match(l)
        if not m:
            continue

        dep_rel = m.group('dep_rel')  # relative to src_dir or absolute
        dep_src = path.abspath(path.join(src_dir, dep_rel))
        if path.isabs(dep_rel):
            # Not a relative path, check if it's absolute if we have been
            # provided with a sphinx source directory root
            if not src_root:
                print("Absolute path to file:", dep_rel, "\n  referenced by:",
                      src_file, "with no --sphinx-src-root", file=sys.stderr)
                continue
            # Make it really relative
            dep_rel = '.' + dep_rel
            dep_src = path.abspath(path.join(src_root, dep_rel))
            if path.isfile(dep_src):
                # File found, but no need to copy it since it's part
                # of Sphinx's top-level source directory
                continue
        if not path.isfile(dep_src):
            print("File not found:", dep_src, "\n  referenced by:",
                  src_file, file=sys.stderr)
            continue

        dep_dst = path.abspath(path.join(dst_dir, dep_rel))
        deps.append(Output(dep_src, dep_dst))

    return deps

def find_content(zephyr_base, src, dest, fnfilter, ignore, src_root):
    # Create a list of Outputs to copy over, and new directories we
    # might need to make to contain them. Don't copy any files or
    # otherwise modify dest.
    outputs = []
    output_dirs = set()
    for dirpath, dirnames, filenames in os.walk(path.join(zephyr_base, src)):
        # Limit the rest of the walk to subdirectories that aren't ignored.
        dirnames[:] = [d for d in dirnames if not
                       path.normpath(path.join(dirpath, d)).startswith(ignore)]

        # If the current directory contains no matching files, keep going.
        sources = fnmatch.filter(filenames, fnfilter)
        if not sources:
            continue

        # There are sources here; track that the output directory
        # needs to exist.
        dst_dir = path.join(dest, path.relpath(dirpath, start=zephyr_base))
        output_dirs.add(path.abspath(dst_dir))

        # Initialize an Output for each source file, as well as any of
        # that file's additional dependencies. Make sure output
        # directories for dependencies are tracked too.
        for src_rel in sources:
            src_abs = path.join(dirpath, src_rel)
            deps = src_deps(zephyr_base, src_abs, dest, src_root)

            for depdir in (path.dirname(d.dst) for d in deps):
                output_dirs.add(depdir)

            outputs.extend(deps)
            outputs.append(Output(src_abs,
                                  path.abspath(path.join(dst_dir, src_rel))))

    return Content(outputs, output_dirs)

def extract_content(content):
    # Ensure each output subdirectory exists.
    for d in content.output_dirs:
        os.makedirs(d, exist_ok=True)

    # Create each output file. Use copy2() to avoid updating
    # modification times unnecessarily, as this triggers documentation
    # rebuilds.
    for output in content.outputs:
        shutil.copy2(output.src, output.dst)

def main():
    parser = argparse.ArgumentParser(
        description='''Recursively copy documentation files from ZEPHYR_BASE to
        a destination folder, along with files referenced in those .rst files
        by a configurable list of directives: {}. The ZEPHYR_BASE environment
        variable is used to determine source directories to copy files
        from.'''.format(DIRECTIVES))

    parser.add_argument('--outputs',
                        help='If given, save input/output files to this path')
    parser.add_argument('--just-outputs', action='store_true',
                        help='''Skip extraction and just list outputs.
                        Cannot be given without --outputs.''')
    parser.add_argument('--ignore', action='append',
                        help='''Source directories to ignore when copying
                        files. This may be given multiple times.''')
    parser.add_argument('--sphinx-src-root',
                        help='''If given, absolute paths for dependencies are
                        resolved using this root, which is the Sphinx top-level
                        source directory as passed to sphinx-build.''')

    parser.add_argument('content_config', nargs='+',
                        help='''A glob:source:destination specification
                        for content to extract. The "glob" is a documentation
                        file name pattern to include, "source" is a source
                        directory to search for such files in, and
                        "destination" is the directory to copy it into.''')
    args = parser.parse_args()

    if "ZEPHYR_BASE" not in os.environ:
        sys.exit("ZEPHYR_BASE environment variable undefined.")
    zephyr_base = os.environ["ZEPHYR_BASE"]

    if not args.ignore:
        ignore = ()
    else:
        ignore = tuple(path.normpath(ign) for ign in args.ignore)

    if args.just_outputs and not args.outputs:
        sys.exit('--just-outputs cannot be given without --outputs')

    content_config = [cfg.split(':', 2) for cfg in args.content_config]
    outputs = set()
    for fnfilter, source, dest in content_config:
        content = find_content(zephyr_base, source, dest, fnfilter, ignore,
                               args.sphinx_src_root)
        if not args.just_outputs:
            extract_content(content)
        outputs |= set(content.outputs)
    if args.outputs:
        with open(args.outputs, 'w') as f:
            for o in outputs:
                print(o.src, file=f, end='\n')
                print(o.dst, file=f, end='\n')

if __name__ == "__main__":
    main()
<commit_msg>doc: extract_content.py: Suppress spurious pylint warning<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2018, Foundries.io Ltd
# Copyright (c) 2018, Nordic Semiconductor ASA
# Copyright (c) 2017, Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

# Internal script used by the documentation's build system to create
# the "final" docs tree which is then compiled by Sphinx.
#
# This works around the fact that Sphinx needs a single documentation
# root directory, while Zephyr's documentation files are spread around
# the tree.

import argparse
import collections
import fnmatch
import os
from os import path
import re
import shutil
import sys

# directives to parse for included files
DIRECTIVES = ["figure", "include", "image", "literalinclude"]

# A simple namedtuple for a generated output file.
#
# - src: source file, what file should be copied (in source directory)
# - dst: destination file, path it should be copied to (in build directory)
Output = collections.namedtuple('Output', 'src dst')

# Represents the content which must be extracted from the Zephyr tree,
# as well as the output directories needed to contain it.
#
# - outputs: list of Output objects for extracted content.
# - output_dirs: set of directories which must exist to contain
#                output destination files.
Content = collections.namedtuple('Content', 'outputs output_dirs')

def src_deps(zephyr_base, src_file, dest, src_root):
    # - zephyr_base: the ZEPHYR_BASE directory containing src_file
    # - src_file: path to a source file in the documentation
    # - dest: path to the top-level output/destination directory
    # - src_root: path to the Sphinx top-level source directory
    #
    # Return a list of Output objects which contain src_file's
    # additional dependencies, as they should be copied into
    # dest. Output paths inside dest are based on each
    # dependency's relative path from zephyr_base.

    # Inspect only .rst files for directives referencing other files
    # we'll need to copy (as configured in the DIRECTIVES variable)
    if not src_file.endswith(".rst"):
        return []

    # Load the file's contents, bailing on decode errors.
    try:
        with open(src_file, encoding="utf-8") as f:
            content = [x.strip() for x in f.readlines()]
    except UnicodeDecodeError as e:
        # pylint: disable=unsubscriptable-object
        sys.stderr.write(
            "Malformed {} in {}\n"
            "  Context: {}\n"
            "  Problematic data: {}\n"
            "  Reason: {}\n".format(
                e.encoding, src_file,
                e.object[max(e.start - 40, 0):e.end + 40],
                e.object[e.start:e.end],
                e.reason))
        return []

    # Source file's directory.
    src_dir = path.dirname(src_file)
    # Destination directory for any dependencies.
    dst_dir = path.join(dest, path.relpath(src_dir, start=zephyr_base))

    # Find directives in the content which imply additional
    # dependencies. We assume each such directive takes a single
    # argument, which is a (relative) path to the additional
    # dependency file.
    directives = "|".join(DIRECTIVES)
    pattern = re.compile(r"\.\.\s+(?P<directive>%s)::\s+(?P<dep_rel>.*)" %
                         directives)
    deps = []
    for l in content:
        m = pattern.match(l)
        if not m:
            continue

        dep_rel = m.group('dep_rel')  # relative to src_dir or absolute
        dep_src = path.abspath(path.join(src_dir, dep_rel))
        if path.isabs(dep_rel):
            # Not a relative path, check if it's absolute if we have been
            # provided with a sphinx source directory root
            if not src_root:
                print("Absolute path to file:", dep_rel, "\n  referenced by:",
                      src_file, "with no --sphinx-src-root", file=sys.stderr)
                continue
            # Make it really relative
            dep_rel = '.' + dep_rel
            dep_src = path.abspath(path.join(src_root, dep_rel))
            if path.isfile(dep_src):
                # File found, but no need to copy it since it's part
                # of Sphinx's top-level source directory
                continue
        if not path.isfile(dep_src):
            print("File not found:", dep_src, "\n  referenced by:",
                  src_file, file=sys.stderr)
            continue

        dep_dst = path.abspath(path.join(dst_dir, dep_rel))
        deps.append(Output(dep_src, dep_dst))

    return deps

def find_content(zephyr_base, src, dest, fnfilter, ignore, src_root):
    # Create a list of Outputs to copy over, and new directories we
    # might need to make to contain them. Don't copy any files or
    # otherwise modify dest.
    outputs = []
    output_dirs = set()
    for dirpath, dirnames, filenames in os.walk(path.join(zephyr_base, src)):
        # Limit the rest of the walk to subdirectories that aren't ignored.
        dirnames[:] = [d for d in dirnames if not
                       path.normpath(path.join(dirpath, d)).startswith(ignore)]

        # If the current directory contains no matching files, keep going.
        sources = fnmatch.filter(filenames, fnfilter)
        if not sources:
            continue

        # There are sources here; track that the output directory
        # needs to exist.
        dst_dir = path.join(dest, path.relpath(dirpath, start=zephyr_base))
        output_dirs.add(path.abspath(dst_dir))

        # Initialize an Output for each source file, as well as any of
        # that file's additional dependencies. Make sure output
        # directories for dependencies are tracked too.
        for src_rel in sources:
            src_abs = path.join(dirpath, src_rel)
            deps = src_deps(zephyr_base, src_abs, dest, src_root)

            for depdir in (path.dirname(d.dst) for d in deps):
                output_dirs.add(depdir)

            outputs.extend(deps)
            outputs.append(Output(src_abs,
                                  path.abspath(path.join(dst_dir, src_rel))))

    return Content(outputs, output_dirs)

def extract_content(content):
    # Ensure each output subdirectory exists.
    for d in content.output_dirs:
        os.makedirs(d, exist_ok=True)

    # Create each output file. Use copy2() to avoid updating
    # modification times unnecessarily, as this triggers documentation
    # rebuilds.
    for output in content.outputs:
        shutil.copy2(output.src, output.dst)

def main():
    parser = argparse.ArgumentParser(
        description='''Recursively copy documentation files from ZEPHYR_BASE to
        a destination folder, along with files referenced in those .rst files
        by a configurable list of directives: {}. The ZEPHYR_BASE environment
        variable is used to determine source directories to copy files
        from.'''.format(DIRECTIVES))

    parser.add_argument('--outputs',
                        help='If given, save input/output files to this path')
    parser.add_argument('--just-outputs', action='store_true',
                        help='''Skip extraction and just list outputs.
                        Cannot be given without --outputs.''')
    parser.add_argument('--ignore', action='append',
                        help='''Source directories to ignore when copying
                        files. This may be given multiple times.''')
    parser.add_argument('--sphinx-src-root',
                        help='''If given, absolute paths for dependencies are
                        resolved using this root, which is the Sphinx top-level
                        source directory as passed to sphinx-build.''')

    parser.add_argument('content_config', nargs='+',
                        help='''A glob:source:destination specification
                        for content to extract. The "glob" is a documentation
                        file name pattern to include, "source" is a source
                        directory to search for such files in, and
                        "destination" is the directory to copy it into.''')
    args = parser.parse_args()

    if "ZEPHYR_BASE" not in os.environ:
        sys.exit("ZEPHYR_BASE environment variable undefined.")
    zephyr_base = os.environ["ZEPHYR_BASE"]

    if not args.ignore:
        ignore = ()
    else:
        ignore = tuple(path.normpath(ign) for ign in args.ignore)

    if args.just_outputs and not args.outputs:
        sys.exit('--just-outputs cannot be given without --outputs')

    content_config = [cfg.split(':', 2) for cfg in args.content_config]
    outputs = set()
    for fnfilter, source, dest in content_config:
        content = find_content(zephyr_base, source, dest, fnfilter, ignore,
                               args.sphinx_src_root)
        if not args.just_outputs:
            extract_content(content)
        outputs |= set(content.outputs)
    if args.outputs:
        with open(args.outputs, 'w') as f:
            for o in outputs:
                print(o.src, file=f, end='\n')
                print(o.dst, file=f, end='\n')

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from flask import render_template, jsonify
from flask.views import View

from app import app
from models import FHGeneric, Result
from models import Http404Error, Http500Error

### ERRORS ###

def deal_error(error, http=404, mode='html'):
    if http == 404:
        return deal_404_error(error, mode)
    elif http == 500:
        return deal_500_error(error, mode)
    else:
        raise Exception("Unimplemented HTTP error: %s" % str(http))

def deal_404_error(error, mode='html'):
    if mode == 'json':
        return jsonify(dict(error=404))
    else:
        return render_template('404.html'), 404

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

def deal_500_error(error, mode='html'):
    """ logs a 500 error and returns the correct template """
    # app.logger.error(error)

    if mode == 'json':
        return jsonify(dict(error=500))
    else:
        return render_template('500.html'), 500

@app.errorhandler(500)
def server_error(e):
    return render_template('500.html'), 500

### GENERAL VIEW HANDLING ###

class GeneralView(View):
    def __init__(self, render_func=jsonify, err_func=lambda *x: x):
        self.render_func = render_func
        self.err_func = err_func

    def dispatch_request(self, **kwargs):
        try:
            context = self.get_objects(**kwargs)
            return self.render_func(**context)
        except Http500Error as e:
            return self.err_func(e, http=500)
        except Http404Error as e:
            return self.err_func(e, http=404)

### INDEX ###

class IndexView(GeneralView):
    def get_objects(self):
        return dict()
        # meta_info = MetaInfo()
        # return dict(generators = meta_info.get_generators(),
        #             suts = meta_info.get_suts(),
        #             analyses = meta_info.get_analyses())

app.add_url_rule('/', view_func=IndexView.as_view(
        'index_html',
        render_func=lambda **kwargs: render_template('index.html', **kwargs),
        err_func=lambda e, **kwargs: deal_error(e, mode='html', **kwargs)
        ))

app.add_url_rule('/api/', view_func=IndexView.as_view(
        'index_json',
        render_func=jsonify,
        err_func=lambda e, **kwargs: deal_error(e, mode='json', **kwargs)
        ))

### RESULT ###

class ResultView(GeneralView):
    def get_objects(self, id=1):
        #result = Result_app(id).infos()
        #return dict(id = result.id,
        #            message = result.message)
        result = FHGeneric(Result)
        return result.id(id)

app.add_url_rule('/result/<int:id>/', view_func=ResultView.as_view(
        'result_html',
        render_func=lambda **kwargs: render_template('result.html', **kwargs),
        err_func=lambda e, **kwargs: deal_error(e, mode='html', **kwargs)
        ))

app.add_url_rule('/api/result/<int:id>/', view_func=ResultView.as_view(
        'result_json',
        render_func=jsonify,
        err_func=lambda e, **kwargs: deal_error(e, mode='json', **kwargs)
        ))

<commit_msg>views.py: generator routes for html/json<commit_after>from flask import render_template, jsonify
from flask.views import View

from app import app
from models import FHGeneric, Result, Generator_app
from models import Http404Error, Http500Error

### ERRORS ###

def deal_error(error, http=404, mode='html'):
    if http == 404:
        return deal_404_error(error, mode)
    elif http == 500:
        return deal_500_error(error, mode)
    else:
        raise Exception("Unimplemented HTTP error: %s" % str(http))

def deal_404_error(error, mode='html'):
    if mode == 'json':
        return jsonify(dict(error=404))
    else:
        return render_template('404.html'), 404

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

def deal_500_error(error, mode='html'):
    """ logs a 500 error and returns the correct template """
    # app.logger.error(error)

    if mode == 'json':
        return jsonify(dict(error=500))
    else:
        return render_template('500.html'), 500

@app.errorhandler(500)
def server_error(e):
    return render_template('500.html'), 500

### GENERAL VIEW HANDLING ###

class GeneralView(View):
    def __init__(self, render_func=jsonify, err_func=lambda *x: x, **kwargs):
        self.render_func = render_func
        self.err_func = err_func
        for kwarg in kwargs:
            setattr(self, kwarg, kwargs[kwarg])

    def dispatch_request(self, **kwargs):
        try:
            context = self.get_objects(**kwargs)
            return self.render_func(**context)
        except Http500Error as e:
            return self.err_func(e, http=500)
        except Http404Error as e:
            return self.err_func(e, http=404)

### INDEX ###

class IndexView(GeneralView):
    def get_objects(self):
        return dict()
        # meta_info = MetaInfo()
        # return dict(generators = meta_info.get_generators(),
        #             suts = meta_info.get_suts(),
        #             analyses = meta_info.get_analyses())

app.add_url_rule('/', view_func=IndexView.as_view(
        'index_html',
        render_func=lambda **kwargs: render_template('index.html', **kwargs),
        err_func=lambda e, **kwargs: deal_error(e, mode='html', **kwargs)
        ))

app.add_url_rule('/api/', view_func=IndexView.as_view(
        'index_json',
        render_func=jsonify,
        err_func=lambda e, **kwargs: deal_error(e, mode='json', **kwargs)
        ))

### FIREHOSE ELEMENT/LIST ###

class ElemView(GeneralView):
    def get_objects(self, id=None, **kwargs):
        obj = self.class_()
        result = obj.id(id)
        return dict(result=result, firehose_type=obj.get_class())

class ListView(GeneralView):
    def get_objects(self):
        obj = self.class_()
        result = obj.all()
        return dict(result=result, firehose_type=obj.get_class())

# GENERATOR # HTML
app.add_url_rule('/view/generator/<int:id>/', view_func=ElemView.as_view(
        'generator_elem_html',
        class_=Generator_app,
        render_func=lambda **kwargs: render_template('elem.html', **kwargs),
        err_func=lambda e, **kwargs: deal_error(e, mode='html', **kwargs)
        ))

app.add_url_rule('/view/generator/', view_func=ListView.as_view(
        'generator_list_html',
        class_=Generator_app,
        render_func=lambda **kwargs: render_template('list.html', **kwargs),
        err_func=lambda e, **kwargs: deal_error(e, mode='html', **kwargs)
        ))

# GENERATOR # JSON
app.add_url_rule('/api/view/generator/<int:id>/', view_func=ElemView.as_view(
        'generator_elem_json',
        class_=Generator_app,
        render_func=jsonify,
        err_func=lambda e, **kwargs: deal_error(e, mode='json', **kwargs)
        ))

app.add_url_rule('/api/view/generator/', view_func=ListView.as_view(
        'generator_list_json',
        class_=Generator_app,
        render_func=jsonify,
        err_func=lambda e, **kwargs: deal_error(e, mode='json', **kwargs)
        ))
<|endoftext|>


<commit_before>import os
import csv

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    from urllib.parse import urljoin
    from urllib.request import urlopen
except ImportError:
    # for Python 2.7 compatibility
    from urlparse import urljoin
    from urllib2 import urlopen

from django.shortcuts import render
from django.views import View
from django.http import HttpResponse
from django.conf import settings

class DataFileView(View):
    file_name = None

    def get(self, request):
        csv = self._get_csv()
        return HttpResponse(csv)

    def _get_csv(self):
        try:
            url = urljoin(getattr(settings, 'CSV_ROOT', None), self.file_name)
            response = urlopen(url)
            data = response.read()
        except ValueError:
            url = urljoin('file://', getattr(settings, 'CSV_ROOT', None))
            url = urljoin(url, self.file_name)
            response = urlopen(url)
            data = response.read()
        except Exception as err:
            data = "Error {}: {}".format(err.errno, err.reason)

        si = StringIO()
        cw = csv.writer(si)
        # csv.reader has to take in string not bytes...
        csv_reader = csv.reader(data.decode("utf-8").splitlines())

        # csv_data = [line.split(b",") for line in data.splitlines()][0]
        header = [str.lower() for str in csv_reader.next()]
        cw.writerow(header)
        # Columns we have to scrub out an & (note double quotes are included)
        # because thats how it is formatted in the csv files...
        scrub = ['major_path', 'code', 'key']
        check_index = []
        for s in scrub:
            if s in header:
                check_index.append(header.index(s))

        if len(check_index) == 0:
            for row in csv_reader:
                cw.writerow(row)
            return si.getvalue().strip('\r\n')
        else:
            for row in csv_reader:
                for index in check_index:
                    row[index] = row[index].replace(" ", "-")
                    row[index] = row[index].replace("&", "_AND_")
                    row[index] = row[index].replace(":", "_")
                cw.writerow(row)
            return si.getvalue().strip('\r\n')

class MajorCourse(DataFileView):
    file_name = "Majors_and_Courses.csv"

class DataMap(DataFileView):
    file_name = "Data_Map.csv"

class StudentData(DataFileView):
    file_name = "Student_Data_All_Majors.csv"

class StatusLookup(DataFileView):
    file_name = "Status_Lookup.csv"
<commit_msg>Changed csv reader function call to be python 3 compatible<commit_after>import os
import csv

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    from urllib.parse import urljoin
    from urllib.request import urlopen
except ImportError:
    # for Python 2.7 compatibility
    from urlparse import urljoin
    from urllib2 import urlopen

from django.shortcuts import render
from django.views import View
from django.http import HttpResponse
from django.conf import settings

class DataFileView(View):
    file_name = None

    def get(self, request):
        csv = self._get_csv()
        return HttpResponse(csv)

    def _get_csv(self):
        try:
            url = urljoin(getattr(settings, 'CSV_ROOT', None), self.file_name)
            response = urlopen(url)
            data = response.read()
        except ValueError:
            url = urljoin('file://', getattr(settings, 'CSV_ROOT', None))
            url = urljoin(url, self.file_name)
            response = urlopen(url)
            data = response.read()
        except Exception as err:
            data = "Error {}: {}".format(err.errno, err.reason)

        si = StringIO()
        cw = csv.writer(si)
        # csv.reader has to take in string not bytes...
        csv_reader = csv.reader(data.decode("utf-8").splitlines())

        # csv_data = [line.split(b",") for line in data.splitlines()][0]
        header = [str.lower() for str in next(csv_reader)]
        cw.writerow(header)
        # Columns we have to scrub out an & (note double quotes are included)
        # because thats how it is formatted in the csv files...
        scrub = ['major_path', 'code', 'key']
        check_index = []
        for s in scrub:
            if s in header:
                check_index.append(header.index(s))

        if len(check_index) == 0:
            for row in csv_reader:
                cw.writerow(row)
            return si.getvalue().strip('\r\n')
        else:
            for row in csv_reader:
                for index in check_index:
                    row[index] = row[index].replace(" ", "-")
                    row[index] = row[index].replace("&", "_AND_")
                    row[index] = row[index].replace(":", "_")
                cw.writerow(row)
            return si.getvalue().strip('\r\n')

class MajorCourse(DataFileView):
    file_name = "Majors_and_Courses.csv"

class DataMap(DataFileView):
    file_name = "Data_Map.csv"

class StudentData(DataFileView):
    file_name = "Student_Data_All_Majors.csv"

class StatusLookup(DataFileView):
    file_name = "Status_Lookup.csv"
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2018, Foundries.io Ltd
# Copyright (c) 2018, Nordic Semiconductor ASA
# Copyright (c) 2017, Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

# Internal script used by the documentation's build system to create
# the "final" docs tree which is then compiled by Sphinx.
#
# This works around the fact that Sphinx needs a single documentation
# root directory, while Zephyr's documentation files are spread around
# the tree.

import argparse
import collections
import fnmatch
import os
from os import path
import re
import shutil
import sys

# directives to parse for included files
DIRECTIVES = ["figure", "include", "image", "literalinclude"]

# A simple namedtuple for a generated output file.
#
# - src: source file, what file should be copied (in source directory)
# - dst: destination file, path it should be copied to (in build directory)
Output = collections.namedtuple('Output', 'src dst')

# Represents the content which must be extracted from the Zephyr tree,
# as well as the output directories needed to contain it.
#
# - outputs: list of Output objects for extracted content.
# - output_dirs: set of directories which must exist to contain
#                output destination files.
Content = collections.namedtuple('Content', 'outputs output_dirs')

def src_deps(zephyr_base, src_file, dest, src_root):
    # - zephyr_base: the ZEPHYR_BASE directory containing src_file
    # - src_file: path to a source file in the documentation
    # - dest: path to the top-level output/destination directory
    # - src_root: path to the Sphinx top-level source directory
    #
    # Return a list of Output objects which contain src_file's
    # additional dependencies, as they should be copied into
    # dest. Output paths inside dest are based on each
    # dependency's relative path from zephyr_base.

    # Inspect only .rst files for directives referencing other files
    # we'll need to copy (as configured in the DIRECTIVES variable)
    if not src_file.endswith(".rst"):
        return []

    # Load the file's contents, bailing on decode errors.
    try:
        with open(src_file, encoding="utf-8") as f:
            content = f.read()
    except UnicodeDecodeError as e:
        # pylint: disable=unsubscriptable-object
        sys.stderr.write(
            "Malformed {} in {}\n"
            "  Context: {}\n"
            "  Problematic data: {}\n"
            "  Reason: {}\n".format(
                e.encoding, src_file,
                e.object[max(e.start - 40, 0):e.end + 40],
                e.object[e.start:e.end],
                e.reason))
        return []

    # Source file's directory.
    src_dir = path.dirname(src_file)
    # Destination directory for any dependencies.
    dst_dir = path.join(dest, path.relpath(src_dir, start=zephyr_base))

    # Find directives in the content which imply additional
    # dependencies. We assume each such directive takes a single
    # argument, which is a (relative) path to the additional
    # dependency file.
    directives = "|".join(DIRECTIVES)
    pattern = re.compile(r"\.\.\s+(?P<directive>%s)::\s+(?P<dep_rel>[^\s]+)" %
                         directives)
    deps = []
    for m in pattern.finditer(content):
        dep_rel = m.group('dep_rel')  # relative to src_dir or absolute
        dep_src = path.abspath(path.join(src_dir, dep_rel))
        if path.isabs(dep_rel):
            # Not a relative path, check if it's absolute if we have been
            # provided with a sphinx source directory root
            if not src_root:
                print("Absolute path to file:", dep_rel, "\n  referenced by:",
                      src_file, "with no --sphinx-src-root", file=sys.stderr)
                continue
            # Make it really relative
            dep_rel = '.' + dep_rel
            dep_src = path.abspath(path.join(src_root, dep_rel))
            if path.isfile(dep_src):
                # File found, but no need to copy it since it's part
                # of Sphinx's top-level source directory
                continue
        if not path.isfile(dep_src):
            print("File not found:", dep_src, "\n  referenced by:",
                  src_file, file=sys.stderr)
            continue

        dep_dst = path.abspath(path.join(dst_dir, dep_rel))
        deps.append(Output(dep_src, dep_dst))

    return deps

def find_content(zephyr_base, src, dest, fnfilter, ignore, src_root):
    # Create a list of Outputs to copy over, and new directories we
    # might need to make to contain them. Don't copy any files or
    # otherwise modify dest.
    outputs = []
    output_dirs = set()
    for dirpath, dirnames, filenames in os.walk(path.join(zephyr_base, src)):
        # Limit the rest of the walk to subdirectories that aren't ignored.
        dirnames[:] = [d for d in dirnames if not
                       path.normpath(path.join(dirpath, d)).startswith(ignore)]

        # Exclude (other) build directories. They may contain previous
        # output from ourselves!
        dirnames[:] = [d for d in dirnames if not
                       path.exists(path.join(dirpath, d, 'CMakeCache.txt'))]

        # If the current directory contains no matching files, keep going.
        sources = fnmatch.filter(filenames, fnfilter)
        if not sources:
            continue

        # There are sources here; track that the output directory
        # needs to exist.
        dst_dir = path.join(dest, path.relpath(dirpath, start=zephyr_base))
        output_dirs.add(path.abspath(dst_dir))

        # Initialize an Output for each source file, as well as any of
        # that file's additional dependencies. Make sure output
        # directories for dependencies are tracked too.
        for src_rel in sources:
            src_abs = path.join(dirpath, src_rel)
            deps = src_deps(zephyr_base, src_abs, dest, src_root)

            for depdir in (path.dirname(d.dst) for d in deps):
                output_dirs.add(depdir)

            outputs.extend(deps)
            outputs.append(Output(src_abs,
                                  path.abspath(path.join(dst_dir, src_rel))))

    return Content(outputs, output_dirs)

def extract_content(content):
    # Ensure each output subdirectory exists.
    for d in content.output_dirs:
        os.makedirs(d, exist_ok=True)

    # Create each output file. Use copy2() to avoid updating
    # modification times unnecessarily, as this triggers documentation
    # rebuilds.
    for output in content.outputs:
        shutil.copy2(output.src, output.dst)

def main():
    parser = argparse.ArgumentParser(
        description='''Recursively copy documentation files from ZEPHYR_BASE to
        a destination folder, along with files referenced in those .rst files
        by a configurable list of directives: {}. The ZEPHYR_BASE environment
        variable is used to determine source directories to copy files
        from.'''.format(DIRECTIVES))

    parser.add_argument('--outputs',
                        help='If given, save input/output files to this path')
    parser.add_argument('--just-outputs', action='store_true',
                        help='''Skip extraction and just list outputs.
                        Cannot be given without --outputs.''')
    parser.add_argument('--ignore', action='append',
                        help='''Source directories to ignore when copying
                        files. This may be given multiple times.''')
    parser.add_argument('--sphinx-src-root',
                        help='''If given, absolute paths for dependencies are
                        resolved using this root, which is the Sphinx top-level
                        source directory as passed to sphinx-build.''')

    parser.add_argument('content_config', nargs='+',
                        help='''A glob:source:destination specification
                        for content to extract. The "glob" is a documentation
                        file name pattern to include, "source" is a source
                        directory to search for such files in, and
                        "destination" is the directory to copy it into.''')
    args = parser.parse_args()

    if "ZEPHYR_BASE" not in os.environ:
        sys.exit("ZEPHYR_BASE environment variable undefined.")
    zephyr_base = os.environ["ZEPHYR_BASE"]

    if not args.ignore:
        ignore = ()
    else:
        ignore = tuple(path.normpath(ign) for ign in args.ignore)

    if args.just_outputs and not args.outputs:
        sys.exit('--just-outputs cannot be given without --outputs')

    content_config = [cfg.split(':', 2) for cfg in args.content_config]
    outputs = set()
    for fnfilter, source, dest in content_config:
        content = find_content(zephyr_base, source, dest, fnfilter, ignore,
                               args.sphinx_src_root)
        if not args.just_outputs:
            extract_content(content)
        outputs |= set(content.outputs)
    if args.outputs:
        with open(args.outputs, 'w') as f:
            for o in outputs:
                print(o.src, file=f, end='\n')
                print(o.dst, file=f, end='\n')

if __name__ == "__main__":
    main()
<commit_msg>doc: scripts: fix parsing directive inside literal<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2018, Foundries.io Ltd
# Copyright (c) 2018, Nordic Semiconductor ASA
# Copyright (c) 2017, Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

# Internal script used by the documentation's build system to create
# the "final" docs tree which is then compiled by Sphinx.
#
# This works around the fact that Sphinx needs a single documentation
# root directory, while Zephyr's documentation files are spread around
# the tree.

import argparse
import collections
import fnmatch
import os
from os import path
import re
import shutil
import sys

# directives to parse for included files
DIRECTIVES = ["figure", "include", "image", "literalinclude"]

# A simple namedtuple for a generated output file.
#
# - src: source file, what file should be copied (in source directory)
# - dst: destination file, path it should be copied to (in build directory)
Output = collections.namedtuple('Output', 'src dst')

# Represents the content which must be extracted from the Zephyr tree,
# as well as the output directories needed to contain it.
#
# - outputs: list of Output objects for extracted content.
# - output_dirs: set of directories which must exist to contain
#                output destination files.
Content = collections.namedtuple('Content', 'outputs output_dirs')

def src_deps(zephyr_base, src_file, dest, src_root):
    # - zephyr_base: the ZEPHYR_BASE directory containing src_file
    # - src_file: path to a source file in the documentation
    # - dest: path to the top-level output/destination directory
    # - src_root: path to the Sphinx top-level source directory
    #
    # Return a list of Output objects which contain src_file's
    # additional dependencies, as they should be copied into
    # dest. Output paths inside dest are based on each
    # dependency's relative path from zephyr_base.

    # Inspect only .rst files for directives referencing other files
    # we'll need to copy (as configured in the DIRECTIVES variable)
    if not src_file.endswith(".rst"):
        return []

    # Load the file's contents, bailing on decode errors.
    try:
        with open(src_file, encoding="utf-8") as f:
            content = f.read()
    except UnicodeDecodeError as e:
        # pylint: disable=unsubscriptable-object
        sys.stderr.write(
            "Malformed {} in {}\n"
            "  Context: {}\n"
            "  Problematic data: {}\n"
            "  Reason: {}\n".format(
                e.encoding, src_file,
                e.object[max(e.start - 40, 0):e.end + 40],
                e.object[e.start:e.end],
                e.reason))
        return []

    # Source file's directory.
    src_dir = path.dirname(src_file)
    # Destination directory for any dependencies.
    dst_dir = path.join(dest, path.relpath(src_dir, start=zephyr_base))

    # Find directives in the content which imply additional
    # dependencies. We assume each such directive takes a single
    # argument, which is a (relative) path to the additional
    # dependency file.
    directives = "|".join(DIRECTIVES)
    pattern = re.compile(r"\.\.\s+(?P<directive>%s)::\s+(?P<dep_rel>[^\s`]+)" %
                         directives)
    deps = []
    for m in pattern.finditer(content):
        dep_rel = m.group('dep_rel')  # relative to src_dir or absolute
        dep_src = path.abspath(path.join(src_dir, dep_rel))
        if path.isabs(dep_rel):
            # Not a relative path, check if it's absolute if we have been
            # provided with a sphinx source directory root
            if not src_root:
                print("Absolute path to file:", dep_rel, "\n  referenced by:",
                      src_file, "with no --sphinx-src-root", file=sys.stderr)
                continue
            # Make it really relative
            dep_rel = '.' + dep_rel
            dep_src = path.abspath(path.join(src_root, dep_rel))
            if path.isfile(dep_src):
                # File found, but no need to copy it since it's part
                # of Sphinx's top-level source directory
                continue
        if not path.isfile(dep_src):
            print("File not found:", dep_src, "\n  referenced by:",
                  src_file, file=sys.stderr)
            continue

        dep_dst = path.abspath(path.join(dst_dir, dep_rel))
        deps.append(Output(dep_src, dep_dst))

    return deps

def find_content(zephyr_base, src, dest, fnfilter, ignore, src_root):
    # Create a list of Outputs to copy over, and new directories we
    # might need to make to contain them. Don't copy any files or
    # otherwise modify dest.
    outputs = []
    output_dirs = set()
    for dirpath, dirnames, filenames in os.walk(path.join(zephyr_base, src)):
        # Limit the rest of the walk to subdirectories that aren't ignored.
        dirnames[:] = [d for d in dirnames if not
                       path.normpath(path.join(dirpath, d)).startswith(ignore)]

        # Exclude (other) build directories. They may contain previous
        # output from ourselves!
        dirnames[:] = [d for d in dirnames if not
                       path.exists(path.join(dirpath, d, 'CMakeCache.txt'))]

        # If the current directory contains no matching files, keep going.
        sources = fnmatch.filter(filenames, fnfilter)
        if not sources:
            continue

        # There are sources here; track that the output directory
        # needs to exist.
        dst_dir = path.join(dest, path.relpath(dirpath, start=zephyr_base))
        output_dirs.add(path.abspath(dst_dir))

        # Initialize an Output for each source file, as well as any of
        # that file's additional dependencies. Make sure output
        # directories for dependencies are tracked too.
        for src_rel in sources:
            src_abs = path.join(dirpath, src_rel)
            deps = src_deps(zephyr_base, src_abs, dest, src_root)

            for depdir in (path.dirname(d.dst) for d in deps):
                output_dirs.add(depdir)

            outputs.extend(deps)
            outputs.append(Output(src_abs,
                                  path.abspath(path.join(dst_dir, src_rel))))

    return Content(outputs, output_dirs)

def extract_content(content):
    # Ensure each output subdirectory exists.
    for d in content.output_dirs:
        os.makedirs(d, exist_ok=True)

    # Create each output file. Use copy2() to avoid updating
    # modification times unnecessarily, as this triggers documentation
    # rebuilds.
    for output in content.outputs:
        shutil.copy2(output.src, output.dst)

def main():
    parser = argparse.ArgumentParser(
        description='''Recursively copy documentation files from ZEPHYR_BASE to
        a destination folder, along with files referenced in those .rst files
        by a configurable list of directives: {}. The ZEPHYR_BASE environment
        variable is used to determine source directories to copy files
        from.'''.format(DIRECTIVES))

    parser.add_argument('--outputs',
                        help='If given, save input/output files to this path')
    parser.add_argument('--just-outputs', action='store_true',
                        help='''Skip extraction and just list outputs.
                        Cannot be given without --outputs.''')
    parser.add_argument('--ignore', action='append',
                        help='''Source directories to ignore when copying
                        files. This may be given multiple times.''')
    parser.add_argument('--sphinx-src-root',
                        help='''If given, absolute paths for dependencies are
                        resolved using this root, which is the Sphinx top-level
                        source directory as passed to sphinx-build.''')

    parser.add_argument('content_config', nargs='+',
                        help='''A glob:source:destination specification
                        for content to extract. The "glob" is a documentation
                        file name pattern to include, "source" is a source
                        directory to search for such files in, and
                        "destination" is the directory to copy it into.''')
    args = parser.parse_args()

    if "ZEPHYR_BASE" not in os.environ:
        sys.exit("ZEPHYR_BASE environment variable undefined.")
    zephyr_base = os.environ["ZEPHYR_BASE"]

    if not args.ignore:
        ignore = ()
    else:
        ignore = tuple(path.normpath(ign) for ign in args.ignore)

    if args.just_outputs and not args.outputs:
        sys.exit('--just-outputs cannot be given without --outputs')

    content_config = [cfg.split(':', 2) for cfg in args.content_config]
    outputs = set()
    for fnfilter, source, dest in content_config:
        content = find_content(zephyr_base, source, dest, fnfilter, ignore,
                               args.sphinx_src_root)
        if not args.just_outputs:
            extract_content(content)
        outputs |= set(content.outputs)
    if args.outputs:
        with open(args.outputs, 'w') as f:
            for o in outputs:
                print(o.src, file=f, end='\n')
                print(o.dst, file=f, end='\n')

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from zang.exceptions.zang_exception import ZangException
from zang.configuration.configuration import Configuration
from zang.connectors.connector_factory import ConnectorFactory
from zang.domain.enums.http_method import HttpMethod

from docs.examples.credetnials import sid, authToken
url = 'http://api.zang.io/v2'

configuration = Configuration(sid, authToken, url=url)
applicationsConnector = ConnectorFactory(configuration).applicationsConnector

# view application
try:
    application = applicationsConnector.viewApplication('TestApplicationSid')
    print(application.friendlyName)
except ZangException as ze:
    print(ze)

# list applications
try:
    application = applicationsConnector.listApplications(
        'TestApplication', 0, 33)
    print(application.total)
except ZangException as ze:
    print(ze)

# create application
try:
    application = applicationsConnector.createApplication(
        friendlyName='TestApplication',
        voiceUrl='voiceUrl',
        voiceMethod=HttpMethod.POST,
        voiceFallbackUrl='voiceFallbackUrl',
        voiceFallbackMethod=HttpMethod.GET,
        voiceCallerIdLookup=True,
        smsUrl='smsUrl',
        smsMethod=HttpMethod.POST,
        smsFallbackUrl='smsFallbackUrl',
        smsFallbackMethod=HttpMethod.GET,
        heartbeatUrl='heartbeatUrl',
        heartbeatMethod=HttpMethod.GET,
        statusCallback='statusCallback',
        statusCallbackMethod=HttpMethod.POST,
        hangupCallback='hangupCallback',
        hangupCallbackMethod=HttpMethod.GET)
    print(application.sid)
except ZangException as ze:
    print(ze)

# update application
try:
    application = applicationsConnector.updateApplication(
        applicationSid='TestApplicationSid',
        friendlyName='TestApplication',
        voiceUrl='voiceUrl',
        voiceMethod=HttpMethod.POST,
        voiceFallbackUrl='voiceFallbackUrl',
        voiceFallbackMethod=HttpMethod.GET,
        voiceCallerIdLookup=True,
        smsUrl='smsUrl',
        smsMethod=HttpMethod.POST,
        smsFallbackUrl='smsFallbackUrl',
        smsFallbackMethod=HttpMethod.GET,
        heartbeatUrl='heartbeatUrl',
        heartbeatMethod=HttpMethod.GET,
        statusCallback='statusCallback',
        statusCallbackMethod=HttpMethod.POST,
        hangupCallback='hangupCallback',
        hangupCallbackMethod=HttpMethod.GET)
    print(application)
except ZangException as ze:
    print(ze)

# delete application
try:
    application = applicationsConnector.deleteApplication('TestApplicationSid')
    print(application.sid)
except ZangException as ze:
    print(ze)
<commit_msg>Update applications_example.py<commit_after>from zang.exceptions.zang_exception import ZangException
from zang.configuration.configuration import Configuration
from zang.connectors.connector_factory import ConnectorFactory
from zang.domain.enums.http_method import HttpMethod

from docs.examples.credentials import sid, authToken
url = 'http://api.zang.io/v2'

configuration = Configuration(sid, authToken, url=url)
applicationsConnector = ConnectorFactory(configuration).applicationsConnector

# view application
try:
    application = applicationsConnector.viewApplication('TestApplicationSid')
    print(application.friendlyName)
except ZangException as ze:
    print(ze)

# list applications
try:
    application = applicationsConnector.listApplications(
        'TestApplication', 0, 33)
    print(application.total)
except ZangException as ze:
    print(ze)

# create application
try:
    application = applicationsConnector.createApplication(
        friendlyName='TestApplication',
        voiceUrl='voiceUrl',
        voiceMethod=HttpMethod.POST,
        voiceFallbackUrl='voiceFallbackUrl',
        voiceFallbackMethod=HttpMethod.GET,
        voiceCallerIdLookup=True,
        smsUrl='smsUrl',
        smsMethod=HttpMethod.POST,
        smsFallbackUrl='smsFallbackUrl',
        smsFallbackMethod=HttpMethod.GET,
        heartbeatUrl='heartbeatUrl',
        heartbeatMethod=HttpMethod.GET,
        statusCallback='statusCallback',
        statusCallbackMethod=HttpMethod.POST,
        hangupCallback='hangupCallback',
        hangupCallbackMethod=HttpMethod.GET)
    print(application.sid)
except ZangException as ze:
    print(ze)

# update application
try:
    application = applicationsConnector.updateApplication(
        applicationSid='TestApplicationSid',
        friendlyName='TestApplication',
        voiceUrl='voiceUrl',
        voiceMethod=HttpMethod.POST,
        voiceFallbackUrl='voiceFallbackUrl',
        voiceFallbackMethod=HttpMethod.GET,
        voiceCallerIdLookup=True,
        smsUrl='smsUrl',
        smsMethod=HttpMethod.POST,
        smsFallbackUrl='smsFallbackUrl',
        smsFallbackMethod=HttpMethod.GET,
        heartbeatUrl='heartbeatUrl',
        heartbeatMethod=HttpMethod.GET,
        statusCallback='statusCallback',
        statusCallbackMethod=HttpMethod.POST,
        hangupCallback='hangupCallback',
        hangupCallbackMethod=HttpMethod.GET)
    print(application)
except ZangException as ze:
    print(ze)

# delete application
try:
    application = applicationsConnector.deleteApplication('TestApplicationSid')
    print(application.sid)
except ZangException as ze:
    print(ze)
<|endoftext|>


<commit_before>#! /usr/bin/env python1.5

r"""Convert old ("regex") regular expressions to new syntax ("re").

When imported as a module, there are two functions, with their own
strings:

  convert(s, syntax=None) -- convert a regex regular expression to re syntax

  quote(s) -- return a quoted string literal

When used as a script, read a Python string literal (or any other
expression evaluating to a string) from stdin, and write the
translated expression to stdout as a string literal.  Unless stdout is
a tty, no trailing \n is written to stdout.  This is done so that it
can be used with Emacs C-U M-| (shell-command-on-region with argument
which filters the region through the shell command).

No attempt has been made at coding for performance.

Translation table...

    \(    (     (unless RE_NO_BK_PARENS set)
    \)    )     (unless RE_NO_BK_PARENS set)
    \|    |     (unless RE_NO_BK_VBAR set)
    \<    \b    (not quite the same, but alla...)
    \>    \b    (not quite the same, but alla...)
    \`    \A
    \'    \Z

Not translated...

    .
    ^
    $
    *
    +           (unless RE_BK_PLUS_QM set, then to \+)
    ?           (unless RE_BK_PLUS_QM set, then to \?)
    \
    \b
    \B
    \w
    \W
    \1 ... \9

Special cases...

    Non-printable characters are always replaced by their 3-digit
    escape code (except \t, \n, \r, which use mnemonic escapes)

    Newline is turned into | when RE_NEWLINE_OR is set

XXX To be done...

    [...]     (different treatment of backslashed items?)
    [^...]    (different treatment of backslashed items?)
    ^ $ * + ? (in some error contexts these are probably treated differently)
    \vDD  \DD (in the regex docs but only works when RE_ANSI_HEX set)

"""

import regex
from regex_syntax import * # RE_*

__all__ = ["convert","quote"]

# Default translation table
mastertable = {
    r'\<': r'\b',
    r'\>': r'\b',
    r'\`': r'\A',
    r'\'': r'\Z',
    r'\(': '(',
    r'\)': ')',
    r'\|': '|',
    '(': r'\(',
    ')': r'\)',
    '|': r'\|',
    '\t': r'\t',
    '\n': r'\n',
    '\r': r'\r',
}

def convert(s, syntax=None):
    """Convert a regex regular expression to re syntax.

    The first argument is the regular expression, as a string object,
    just like it would be passed to regex.compile().  (I.e., pass the
    actual string object -- string quotes must already have been
    removed and the standard escape processing has already been done,
    e.g. by eval().)

    The optional second argument is the regex syntax variant to be
    used.  This is an integer mask as passed to regex.set_syntax();
    the flag bits are defined in regex_syntax.  When not specified, or
    when None is given, the current regex syntax mask (as retrieved by
    regex.get_syntax()) is used -- which is 0 by default.

    The return value is a regular expression, as a string object that
    could be passed to re.compile().  (I.e., no string quotes have
    been added -- use quote() below, or repr().)

    The conversion is not always guaranteed to be correct.  More
    syntactical analysis should be performed to detect borderline
    cases and decide what to do with them.  For example, 'x*?' is not
    translated correctly.

    """
    table = mastertable.copy()
    if syntax is None:
        syntax = regex.get_syntax()
    if syntax & RE_NO_BK_PARENS:
        del table[r'\('], table[r'\)']
        del table['('], table[')']
    if syntax & RE_NO_BK_VBAR:
        del table[r'\|']
        del table['|']
    if syntax & RE_BK_PLUS_QM:
        table['+'] = r'\+'
        table['?'] = r'\?'
        table[r'\+'] = '+'
        table[r'\?'] = '?'
    if syntax & RE_NEWLINE_OR:
        table['\n'] = '|'
    res = ""

    i = 0
    end = len(s)
    while i < end:
        c = s[i]
        i = i+1
        if c == '\\':
            c = s[i]
            i = i+1
            key = '\\' + c
            key = table.get(key, key)
            res = res + key
        else:
            c = table.get(c, c)
            res = res + c
    return res

def quote(s, quote=None):
    """Convert a string object to a quoted string literal.

    This is similar to repr() but will return a "raw" string (r'...'
    or r"...") when the string contains backslashes, instead of
    doubling all backslashes.  The resulting string does *not* always
    evaluate to the same string as the original; however it will do
    just the right thing when passed into re.compile().

    The optional second argument forces the string quote; it must be
    a single character which is a valid Python string quote.

    """
    if quote is None:
        q = "'"
        altq = "'"
        if q in s and altq not in s:
            q = altq
    else:
        assert quote in ('"', "'")
        q = quote
    res = q
    for c in s:
        if c == q: c = '\\' + c
        elif c < ' ' or c > '~': c = "\\%03o" % ord(c)
        res = res + c
    res = res + q
    if '\\' in res:
        res = 'r' + res
    return res

def main():
    """Main program -- called when run as a script."""
    import sys
    s = eval(sys.stdin.read())
    sys.stdout.write(quote(convert(s)))
    if sys.stdout.isatty():
        sys.stdout.write("\n")

if __name__ == '__main__':
    main()
<commit_msg>Suppress the warning about regex here.<commit_after>#! /usr/bin/env python1.5

r"""Convert old ("regex") regular expressions to new syntax ("re").

When imported as a module, there are two functions, with their own
strings:

  convert(s, syntax=None) -- convert a regex regular expression to re syntax

  quote(s) -- return a quoted string literal

When used as a script, read a Python string literal (or any other
expression evaluating to a string) from stdin, and write the
translated expression to stdout as a string literal.  Unless stdout is
a tty, no trailing \n is written to stdout.  This is done so that it
can be used with Emacs C-U M-| (shell-command-on-region with argument
which filters the region through the shell command).

No attempt has been made at coding for performance.

Translation table...

    \(    (     (unless RE_NO_BK_PARENS set)
    \)    )     (unless RE_NO_BK_PARENS set)
    \|    |     (unless RE_NO_BK_VBAR set)
    \<    \b    (not quite the same, but alla...)
    \>    \b    (not quite the same, but alla...)
    \`    \A
    \'    \Z

Not translated...

    .
    ^
    $
    *
    +           (unless RE_BK_PLUS_QM set, then to \+)
    ?           (unless RE_BK_PLUS_QM set, then to \?)
    \
    \b
    \B
    \w
    \W
    \1 ... \9

Special cases...

    Non-printable characters are always replaced by their 3-digit
    escape code (except \t, \n, \r, which use mnemonic escapes)

    Newline is turned into | when RE_NEWLINE_OR is set

XXX To be done...

    [...]     (different treatment of backslashed items?)
    [^...]    (different treatment of backslashed items?)
    ^ $ * + ? (in some error contexts these are probably treated differently)
    \vDD  \DD (in the regex docs but only works when RE_ANSI_HEX set)

"""

import warnings
warnings.filterwarnings("ignore", ".* regex .*", DeprecationWarning, __name__,
                        append=1)

import regex
from regex_syntax import * # RE_*

__all__ = ["convert","quote"]

# Default translation table
mastertable = {
    r'\<': r'\b',
    r'\>': r'\b',
    r'\`': r'\A',
    r'\'': r'\Z',
    r'\(': '(',
    r'\)': ')',
    r'\|': '|',
    '(': r'\(',
    ')': r'\)',
    '|': r'\|',
    '\t': r'\t',
    '\n': r'\n',
    '\r': r'\r',
}

def convert(s, syntax=None):
    """Convert a regex regular expression to re syntax.

    The first argument is the regular expression, as a string object,
    just like it would be passed to regex.compile().  (I.e., pass the
    actual string object -- string quotes must already have been
    removed and the standard escape processing has already been done,
    e.g. by eval().)

    The optional second argument is the regex syntax variant to be
    used.  This is an integer mask as passed to regex.set_syntax();
    the flag bits are defined in regex_syntax.  When not specified, or
    when None is given, the current regex syntax mask (as retrieved by
    regex.get_syntax()) is used -- which is 0 by default.

    The return value is a regular expression, as a string object that
    could be passed to re.compile().  (I.e., no string quotes have
    been added -- use quote() below, or repr().)

    The conversion is not always guaranteed to be correct.  More
    syntactical analysis should be performed to detect borderline
    cases and decide what to do with them.  For example, 'x*?' is not
    translated correctly.

    """
    table = mastertable.copy()
    if syntax is None:
        syntax = regex.get_syntax()
    if syntax & RE_NO_BK_PARENS:
        del table[r'\('], table[r'\)']
        del table['('], table[')']
    if syntax & RE_NO_BK_VBAR:
        del table[r'\|']
        del table['|']
    if syntax & RE_BK_PLUS_QM:
        table['+'] = r'\+'
        table['?'] = r'\?'
        table[r'\+'] = '+'
        table[r'\?'] = '?'
    if syntax & RE_NEWLINE_OR:
        table['\n'] = '|'
    res = ""

    i = 0
    end = len(s)
    while i < end:
        c = s[i]
        i = i+1
        if c == '\\':
            c = s[i]
            i = i+1
            key = '\\' + c
            key = table.get(key, key)
            res = res + key
        else:
            c = table.get(c, c)
            res = res + c
    return res

def quote(s, quote=None):
    """Convert a string object to a quoted string literal.

    This is similar to repr() but will return a "raw" string (r'...'
    or r"...") when the string contains backslashes, instead of
    doubling all backslashes.  The resulting string does *not* always
    evaluate to the same string as the original; however it will do
    just the right thing when passed into re.compile().

    The optional second argument forces the string quote; it must be
    a single character which is a valid Python string quote.

    """
    if quote is None:
        q = "'"
        altq = "'"
        if q in s and altq not in s:
            q = altq
    else:
        assert quote in ('"', "'")
        q = quote
    res = q
    for c in s:
        if c == q: c = '\\' + c
        elif c < ' ' or c > '~': c = "\\%03o" % ord(c)
        res = res + c
    res = res + q
    if '\\' in res:
        res = 'r' + res
    return res

def main():
    """Main program -- called when run as a script."""
    import sys
    s = eval(sys.stdin.read())
    sys.stdout.write(quote(convert(s)))
    if sys.stdout.isatty():
        sys.stdout.write("\n")

if __name__ == '__main__':
    main()
<|endoftext|>