<commit_before># -*- coding: utf-8 -*-
# Copyright © 2014—2016 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' Unreal Engine 4 related stuff '''

import os
import platform
import logging

import nimp.build
import nimp.system

def load_config(env):
    ''' Loads Unreal specific configuration values on env before parsing
        command-line arguments '''
    env.is_ue4 = hasattr(env, 'project_type') and env.project_type is 'UE4'
    return True

def load_arguments(env):
    ''' Loads Unreal specific environment parameters. '''
    if hasattr(env, 'project_type') and env.project_type == 'UE4':
        return _ue4_load_arguments(env)

    return True

def build(env):
    ''' Builds an Unreal Engine Project. config and platform arguments should
        be set on environment in order to call this function. You can use
        `nimp.environment.add_arguments` and `nimp.build.add_arguments` to do
        so.'''
    if not _check_for_unreal(env):
        return False
    return _ue4_build(env)

def commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an Unreal Engine commandlet. It can be usefull to run it through
        nimp as OutputDebugString will be redirected to standard output. '''
    if not _check_for_unreal(env):
        return False
    return _ue4_commandlet(env, command, *args, heartbeat = heartbeat)

def is_unreal4_available(env):
    ''' Returns a tuple containing unreal availability and a help text
        giving a reason of why it isn't if it's the case '''
    return env.is_ue4, ('No .nimp.conf configured for Unreal 4 was found in '
                        'this directory or one of its parents. Check that you '
                        'are launching nimp from inside an UE4 project directory'
                        ' and that you have a properly configured .nimp.conf. '
                        'Check documentation for more details on .nimp.conf.')
def _check_for_unreal(env):
    if not env.is_ue4:
        logging.error('This doesn\'t seems to be a supported project type.')
        logging.error('Check that you are launching nimp from an UE project')
        logging.error('directory, and that you have a .nimp.conf file in the')
        logging.error('project root directory (see documentation for further')
        logging.error('details)')
        return False
    return True

def _ue4_build(env):
    assert hasattr(env, 'ue4_config')
    assert env.ue4_config is not None

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        os.environ['UBT_bUseUnityBuild'] = 'false'

    nimp.environment.execute_hook('before_ue4_build', env)

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            logging.error("Error generating UE4 project files")
            return False

    # Default to VS 2015, The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '14'
    try:
        for line in open(env.format(env.solution)):
            if ('# Visual Studio 2012' in line or
                '# Visual Studio 11' in line):
                vs_version = '11'
                break
            if ('# Visual Studio 2013' in line or
                '# Visual Studio 12' in line):
                vs_version = '12'
                break
            if ('# Visual Studio 2015' in line or
                '# Visual Studio 14' in line):
                vs_version = '14'
                break
    except IOError:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    success = True

    # List of tools to build
    tools = [ 'UnrealHeaderTool' ]

    # Some tools are necessary even when not building tools...
    need_ps4devkitutil = False
    need_ps4mapfileutil = env.platform == 'ps4'
    need_xboxonepdbfileutil = env.platform == 'xboxone'

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker',
                   'UnrealPak', ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'SymbolDebugger',
                       'CrashReportClient', ]
            need_ps4devkitutil = True
            need_ps4mapfileutil = True
            need_xboxonepdbfileutil = True

    # Some tools are necessary even when not building tools...
    if need_ps4devkitutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4DevKitUtil/PS4DevKitUtil.csproj'))):
        tools += [ 'PS4DevKitUtil' ]

    if need_ps4mapfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4MapFileUtil/PS4MapFileUtil.Build.cs'))):
        tools += [ 'PS4MapFileUtil' ]

    if need_xboxonepdbfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePDBFileUtil/XboxOnePDBFileUtil.Build.cs'))):
        tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Shipping' if tool == 'CrashReportClient'
                                  else 'Development',
                                  vs_version, 'Build'):
            logging.error("Could not build %s", tool)
            success = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                                  'Any CPU', 'Development', None, vs_version, 'Build'):
            logging.error("Could not build NetworkProfiler")
            success = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            # XXX: try 10 times because of a race condition in there
            for attempt in range(10):
                swarm_success = True
                if nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                                      'Any CPU', 'Development', None, vs_version, 'Build'):
                    break
                logging.error("Could not build UnrealSwarm")
                swarm_success = False

            if not swarm_success:
                success = False

            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                                      'Any CPU', 'Development', None, vs_version, 'Build'):
                logging.error("Could not build SwarmInterface")
                success = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                logging.error("Could not build BootstrapPackagedGame")
                success = False

            tmp = env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln')
            if os.path.exists(nimp.system.sanitize_path(tmp)):
                if not nimp.build.vsbuild(tmp, 'x64', 'Development', None, '11', 'Build') \
                   and not nimp.build.vsbuild(tmp, 'x64', 'Development', None, '14', 'Build'):
                    logging.error("Could not build XboxOnePackageNameUtil")
                    success = False

    if not success:
        return success

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_platform,
                                  env.ue4_config, vs_version, 'Build'):
            success = False

    if env.target == 'editor':
        if env.platform in ['linux', 'mac']:
            project = env.game + 'Editor'
            config = env.ue4_config
        else:
            project = env.game
            config = env.ue4_config + ' Editor'

        if not _ue4_build_project(env, solution, project, env.ue4_platform,
                                  config, vs_version, 'Build'):
            success = False

    nimp.environment.execute_hook('after_ue4_build', env, success)

    return success

def _ue4_commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an UE4 commandlet '''
    if nimp.system.is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [nimp.system.sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % command]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return nimp.system.call_process('.', cmdline, heartbeat = heartbeat) == 0

def _ue4_generate_project(env):
    if nimp.system.is_windows():
        return nimp.system.call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat', '-2015'])
    else:
        return nimp.system.call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if nimp.system.is_windows():
        return nimp.build.vsbuild(sln_file, build_platform, configuration,
                                  project, vs_version, target)

    if platform.system() == 'Darwin':
        host_platform = 'Mac'
    else:
        host_platform = 'Linux'

    return nimp.system.call_process(env.root_dir,
                                    ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (host_platform),
                                     project, build_platform, configuration]) == 0

def _ue4_load_arguments(env):
    ''' Sets some variables for use with unreal 4 '''
    def _get_ue4_config(config):
        configs = { "debug"    : "Debug",
                    "devel"    : "Development",
                    "test"     : "Test",
                    "shipping" : "Shipping", }
        if config not in configs:
            logging.warning('Unsupported UE4 build config “%s”', config)
            return None
        return configs[config]

    def _get_ue4_platform(in_platform):
        platforms = { "ps4"     : "PS4",
                      "xboxone" : "XboxOne",
                      "win64"   : "Win64",
                      "win32"   : "Win32",
                      "linux"   : "Linux",
                      "mac"     : "Mac", }
        if in_platform not in platforms:
            logging.warning('Unsupported UE4 build platform “%s”', in_platform)
            return None
        return platforms[in_platform]

    if hasattr(env, 'platform'):
        env.ue4_platform = '+'.join(map(_get_ue4_platform, env.platform.split('+')))

    if hasattr(env, 'configuration'):
        if env.configuration is None:
            env.configuration = 'devel'
        env.ue4_config = '+'.join(map(_get_ue4_config, env.configuration.split('+')))

    if not hasattr(env, 'target') or env.target is None and env.is_ue4:
        if env.platform in ['win64', 'mac', 'linux']:
            env.target = 'editor'
        else:
            env.target = 'game'

    return True
<commit_msg>Add more user-friendly hook aliases for UE4 builds (prebuild, postbuild).<commit_after># -*- coding: utf-8 -*-
# Copyright © 2014—2016 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' Unreal Engine 4 related stuff '''

import os
import platform
import logging

import nimp.build
import nimp.system

def load_config(env):
    ''' Loads Unreal specific configuration values on env before parsing
        command-line arguments '''
    env.is_ue4 = hasattr(env, 'project_type') and env.project_type is 'UE4'
    return True

def load_arguments(env):
    ''' Loads Unreal specific environment parameters. '''
    if hasattr(env, 'project_type') and env.project_type == 'UE4':
        return _ue4_load_arguments(env)

    return True

def build(env):
    ''' Builds an Unreal Engine Project. config and platform arguments should
        be set on environment in order to call this function. You can use
        `nimp.environment.add_arguments` and `nimp.build.add_arguments` to do
        so.'''
    if not _check_for_unreal(env):
        return False
    return _ue4_build(env)

def commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an Unreal Engine commandlet. It can be usefull to run it through
        nimp as OutputDebugString will be redirected to standard output. '''
    if not _check_for_unreal(env):
        return False
    return _ue4_commandlet(env, command, *args, heartbeat = heartbeat)

def is_unreal4_available(env):
    ''' Returns a tuple containing unreal availability and a help text
        giving a reason of why it isn't if it's the case '''
    return env.is_ue4, ('No .nimp.conf configured for Unreal 4 was found in '
                        'this directory or one of its parents. Check that you '
                        'are launching nimp from inside an UE4 project directory'
                        ' and that you have a properly configured .nimp.conf. '
                        'Check documentation for more details on .nimp.conf.')
def _check_for_unreal(env):
    if not env.is_ue4:
        logging.error('This doesn\'t seems to be a supported project type.')
        logging.error('Check that you are launching nimp from an UE project')
        logging.error('directory, and that you have a .nimp.conf file in the')
        logging.error('project root directory (see documentation for further')
        logging.error('details)')
        return False
    return True

def _ue4_build(env):
    assert hasattr(env, 'ue4_config')
    assert env.ue4_config is not None

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        os.environ['UBT_bUseUnityBuild'] = 'false'

    nimp.environment.execute_hook('before_ue4_build', env)
    nimp.environment.execute_hook('prebuild', env)

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            logging.error("Error generating UE4 project files")
            return False

    # Default to VS 2015, The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '14'
    try:
        for line in open(env.format(env.solution)):
            if ('# Visual Studio 2012' in line or
                '# Visual Studio 11' in line):
                vs_version = '11'
                break
            if ('# Visual Studio 2013' in line or
                '# Visual Studio 12' in line):
                vs_version = '12'
                break
            if ('# Visual Studio 2015' in line or
                '# Visual Studio 14' in line):
                vs_version = '14'
                break
    except IOError:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    success = True

    # List of tools to build
    tools = [ 'UnrealHeaderTool' ]

    # Some tools are necessary even when not building tools...
    need_ps4devkitutil = False
    need_ps4mapfileutil = env.platform == 'ps4'
    need_xboxonepdbfileutil = env.platform == 'xboxone'

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker',
                   'UnrealPak', ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'SymbolDebugger',
                       'CrashReportClient', ]
            need_ps4devkitutil = True
            need_ps4mapfileutil = True
            need_xboxonepdbfileutil = True

    # Some tools are necessary even when not building tools...
    if need_ps4devkitutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4DevKitUtil/PS4DevKitUtil.csproj'))):
        tools += [ 'PS4DevKitUtil' ]

    if need_ps4mapfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4MapFileUtil/PS4MapFileUtil.Build.cs'))):
        tools += [ 'PS4MapFileUtil' ]

    if need_xboxonepdbfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePDBFileUtil/XboxOnePDBFileUtil.Build.cs'))):
        tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Shipping' if tool == 'CrashReportClient'
                                  else 'Development',
                                  vs_version, 'Build'):
            logging.error("Could not build %s", tool)
            success = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                                  'Any CPU', 'Development', None, vs_version, 'Build'):
            logging.error("Could not build NetworkProfiler")
            success = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            # XXX: try 10 times because of a race condition in there
            for attempt in range(10):
                swarm_success = True
                if nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                                      'Any CPU', 'Development', None, vs_version, 'Build'):
                    break
                logging.error("Could not build UnrealSwarm")
                swarm_success = False

            if not swarm_success:
                success = False

            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                                      'Any CPU', 'Development', None, vs_version, 'Build'):
                logging.error("Could not build SwarmInterface")
                success = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                logging.error("Could not build BootstrapPackagedGame")
                success = False

            tmp = env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln')
            if os.path.exists(nimp.system.sanitize_path(tmp)):
                if not nimp.build.vsbuild(tmp, 'x64', 'Development', None, '11', 'Build') \
                   and not nimp.build.vsbuild(tmp, 'x64', 'Development', None, '14', 'Build'):
                    logging.error("Could not build XboxOnePackageNameUtil")
                    success = False

    if not success:
        return success

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_platform,
                                  env.ue4_config, vs_version, 'Build'):
            success = False

    if env.target == 'editor':
        if env.platform in ['linux', 'mac']:
            project = env.game + 'Editor'
            config = env.ue4_config
        else:
            project = env.game
            config = env.ue4_config + ' Editor'

        if not _ue4_build_project(env, solution, project, env.ue4_platform,
                                  config, vs_version, 'Build'):
            success = False

    nimp.environment.execute_hook('after_ue4_build', env, success)
    nimp.environment.execute_hook('postbuild', env, success)

    return success

def _ue4_commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an UE4 commandlet '''
    if nimp.system.is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [nimp.system.sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % command]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return nimp.system.call_process('.', cmdline, heartbeat = heartbeat) == 0

def _ue4_generate_project(env):
    if nimp.system.is_windows():
        return nimp.system.call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat', '-2015'])
    else:
        return nimp.system.call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if nimp.system.is_windows():
        return nimp.build.vsbuild(sln_file, build_platform, configuration,
                                  project, vs_version, target)

    if platform.system() == 'Darwin':
        host_platform = 'Mac'
    else:
        host_platform = 'Linux'

    return nimp.system.call_process(env.root_dir,
                                    ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (host_platform),
                                     project, build_platform, configuration]) == 0

def _ue4_load_arguments(env):
    ''' Sets some variables for use with unreal 4 '''
    def _get_ue4_config(config):
        configs = { "debug"    : "Debug",
                    "devel"    : "Development",
                    "test"     : "Test",
                    "shipping" : "Shipping", }
        if config not in configs:
            logging.warning('Unsupported UE4 build config “%s”', config)
            return None
        return configs[config]

    def _get_ue4_platform(in_platform):
        platforms = { "ps4"     : "PS4",
                      "xboxone" : "XboxOne",
                      "win64"   : "Win64",
                      "win32"   : "Win32",
                      "linux"   : "Linux",
                      "mac"     : "Mac", }
        if in_platform not in platforms:
            logging.warning('Unsupported UE4 build platform “%s”', in_platform)
            return None
        return platforms[in_platform]

    if hasattr(env, 'platform'):
        env.ue4_platform = '+'.join(map(_get_ue4_platform, env.platform.split('+')))

    if hasattr(env, 'configuration'):
        if env.configuration is None:
            env.configuration = 'devel'
        env.ue4_config = '+'.join(map(_get_ue4_config, env.configuration.split('+')))

    if not hasattr(env, 'target') or env.target is None and env.is_ue4:
        if env.platform in ['win64', 'mac', 'linux']:
            env.target = 'editor'
        else:
            env.target = 'game'

    return True
<|endoftext|>


<commit_before>import sys
import os
from .core import ninja_generate
from .tk import ioTK
from .io.read import read_config_files

def main(f_out, args):
    if os.path.splitext(f_out)[1] == ".yaml":
        args = [f_out] + args
        f_out = "build.ninja"
        print("output", f_out)
        print(args)
    if len(args) == 0:
        here = os.path.dirname(os.path.realpath(__file__))
        f_usage = os.path.join(here, 'usage.txt')
        print(ioTK.read_all(f_usage))
        print("Error hint: need at least two command line arguments")
        exit()

    ioTK.save_text(
            f_out,
            ninja_generate(read_config_files(args))
        )
<commit_msg>remove debug print<commit_after>import sys
import os
from .core import ninja_generate
from .tk import ioTK
from .io.read import read_config_files

def main(f_out, args):
    if os.path.splitext(f_out)[1] == ".yaml":
        args = [f_out] + args
        f_out = "build.ninja"
    if len(args) == 0:
        here = os.path.dirname(os.path.realpath(__file__))
        f_usage = os.path.join(here, 'usage.txt')
        print(ioTK.read_all(f_usage))
        print("Error hint: need at least two command line arguments")
        exit()

    ioTK.save_text(
            f_out,
            ninja_generate(read_config_files(args))
        )
<|endoftext|>


<commit_before>from expects.testing import failure
from expects import *
from enerdata.contracts.tariff import *
from datetime import datetime

with description('Create a period'):
    with it('accepts "te"'):
        TariffPeriod('P1', 'te')
    with it('accepts "tp"'):
        TariffPeriod('P1', 'tp')
    with it('fails when is not "te" nor "tp"'):
        with failure:
            TariffPeriod('P1', 'foo')
    with it('should raise an exception if range of hours is invalid'):
        expect(lambda: TariffPeriod('P1', 'te', winter_hours=[
            (0, 12), (11, 23)
        ])).to(raise_error(ValueError, 'Invalid winter hours'))
        expect(lambda: TariffPeriod('P1', 'te', summer_hours=[
            (0, 12), (11, 23)
        ])).to(raise_error(ValueError, 'Invalid summer hours'))

with description('A period'):
    with it('should have a range of hours 24 for winter by default'):
        p1 = TariffPeriod('P1', 'te')
        assert p1.winter_hours == [(0, 24)]
    with it('should be possible to set the range of winter hours in creation'):
        p1 = TariffPeriod('P1', 'te', winter_hours=[(0, 12)])
        assert p1.winter_hours == [(0, 12)]
    with it('should have a range of hours 24 for summer by default'):
        p1 = TariffPeriod('P1', 'te')
        assert p1.summer_hours == [(0, 24)]
    with it('should be possible to set the range of sumer hours in creation'):
        p1 = TariffPeriod('P1', 'te', summer_hours=[(0, 12)])
        assert p1.summer_hours == [(0, 12)]
    with it('has to validate range hours is correct'):
        assert check_range_hours([(0, 12)]) is True
        assert check_range_hours([(0, 12), (12, 24)]) is True
        assert check_range_hours([(-1, 0)]) is False
        assert check_range_hours([(0, 25)]) is False
        assert check_range_hours([(0, 0)]) is False
        assert check_range_hours([(4, 1)]) is False
        assert check_range_hours([(0, 1), (0, 2)]) is False
        assert check_range_hours([(0, 12), (12, 24)]) is True
    with it('should know the total of hours in summer'):
        p1 = TariffPeriod('P1', 'te', summer_hours=[(0, 12), (22, 24)])
        assert p1.total_summer_hours == 14
    with it('should know the total of hours in winter'):
        p1 = TariffPeriod('P1', 'te', winter_hours=[(12, 22)])
        assert p1.total_winter_hours == 10

with context('A tariff'):
    with before.all:
        self.tariff = Tariff('T1')
    with it('periods should be a tuple type'):
        assert isinstance(self.tariff.periods, tuple)
    with it('should return the number of periods of te'):
        self.tariff.periods = (
            TariffPeriod('1', 'te', winter_hours=[(12, 22)], summer_hours=[(13, 23)]),
            TariffPeriod('2', 'te', winter_hours=[(0, 12), (22, 24)], summer_hours=[(0, 13), (23, 24)])
        )
        assert self.tariff.get_number_of_periods() == 2
    with it('should return the periods of energy'):
        assert len(self.tariff.energy_periods) == 2
        assert self.tariff.energy_periods.keys() == ['1', '2']
    with it('should return the periods of power'):
        assert len(self.tariff.power_periods) == 0
        self.tariff.periods += (TariffPeriod('1', 'tp'),)
        assert len(self.tariff.power_periods) == 1
        assert self.tariff.power_periods.keys() == ['1']
    with it('should have 24h of range ours in its energy periods'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('1', 'te', summer_hours=[(12, 22)]),
                TariffPeriod('2', 'te', summer_hours=[(0, 12), (22, 23)])
            )
        expect(set_periods).to(raise_error(ValueError))

    with it('should check range of hours'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('1', 'te', summer_hours=[(13, 23)]),
                TariffPeriod('2', 'te', summer_hours=[(0, 12), (22, 24)])
            )
        expect(set_periods).to(raise_error(ValueError))

    with it('should check range and hours if a holiday period is defined'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('P1', 'te', winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
                TariffPeriod('P2', 'te', winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
                TariffPeriod('P3', 'te', winter_hours=[(0, 8)], summer_hours=[(0, 8)]),
                TariffPeriod('P4', 'te', holiday=True, winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
                TariffPeriod('P5', 'te', holiday=True, winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
                TariffPeriod('P6', 'te', holiday=True, winter_hours=[(0, 8)], summer_hours=[(1, 8)])
            )
        expect(set_periods).to(raise_error(ValueError, 'The sum of hours in summer (in holidays) must be 24h: [(1, 8), (8, 11), (11, 15), (15, 24)]'))

    with it('should find the period by datetime'):
        self.tariff.periods = (
            TariffPeriod('P1', 'te', winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
            TariffPeriod('P2', 'te', winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
            TariffPeriod('P3', 'te', winter_hours=[(0, 8)], summer_hours=[(0, 8)]),
            TariffPeriod('P4', 'te', holiday=True, winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
            TariffPeriod('P5', 'te', holiday=True, winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
            TariffPeriod('P6', 'te', holiday=True, winter_hours=[(0, 8)], summer_hours=[(0, 8)])
        )
        dt = datetime(2015, 12, 24, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P1'
        dt = datetime(2015, 12, 25, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 17, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P5'
        dt = datetime(2015, 12, 27, 1, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P6'
    with it('should allow to check if a set of powers is correct'):
        tari_T20A = T20A()
        expect(lambda: tari_T20A.evaluate_powers([-10])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T20A.evaluate_powers([0])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T20A.evaluate_powers([5])).to(
            raise_error(
                NotNormalizedPower, 'Power 5kW isn\'t a normalized value'
            ))
        assert tari_T20A.evaluate_powers([5.5])
        expect(lambda: tari_T20A.evaluate_powers([5, 7])).to(
            raise_error(IncorrectPowerNumber, 'Expected 1 power(s) and got 2'))
        expect(lambda: tari_T20A.evaluate_powers([100])).to(
            raise_error(IncorrectMaxPower))

        tari_T30A = T30A()
        expect(lambda: tari_T30A.evaluate_powers([-10, -5, 0])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T30A.evaluate_powers([10, 13, 15])).to(
            raise_error(IncorrectMaxPower))
        expect(lambda: tari_T30A.evaluate_powers([10, 13, 16])).to(
            raise_error(
                NotNormalizedPower, 'Power 10kW isn\'t a normalized value'
            ))
        assert tari_T30A.evaluate_powers([2.304, 2.304, 16.454])
        expect(lambda: tari_T30A.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 3 power(s) and got 2'))

        tari_T31A = T31A()
        expect(lambda: tari_T31A.evaluate_powers([-10, -5, 0])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T31A.evaluate_powers([10, 13, 16])).to(
            raise_error(
                NotNormalizedPower, 'Power 10kW isn\'t a normalized value'
            ))
        assert tari_T31A.evaluate_powers([2.304, 2.304, 16.454])
        expect(lambda: tari_T31A.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 3 power(s) and got 2'))
        expect(lambda: tari_T31A.evaluate_powers([16, 20, 16])).to(
            raise_error(NotAscendingPowers))
    with it('should allow to check if a maximum power is correct'):
        tari_T20A = T20A()
        assert not tari_T20A.is_maximum_power_correct(-10)
        assert not tari_T20A.is_maximum_power_correct(0)
        assert tari_T20A.is_maximum_power_correct(7)
        assert tari_T20A.is_maximum_power_correct(10)
        assert not tari_T20A.is_maximum_power_correct(1000)

with context('3.0A tariff'):
    with before.all:
        self.tariff = T30A()
        self.periods = self.tariff.energy_periods
    with it("should return 6 energy periods"):
        assert len(self.periods) == 6
    with it("should contain P1 to P6 periods"):
        assert 'P1' in self.periods.keys()
        assert 'P2' in self.periods.keys()
        assert 'P3' in self.periods.keys()
        assert 'P4' in self.periods.keys()
        assert 'P5' in self.periods.keys()
        assert 'P6' in self.periods.keys()

with description('Getting a tariff by descripion'):

    with it('must return the appropiate tariff and initialize the expected cof'):
        tariffs = [
            ('2.0A', T20A),
            ('2.0DHA', T20DHA),
            ('2.0DHS', T20DHS),
            ('2.1A', T21A),
            ('2.1DHA', T21DHA),
            ('2.1DHS', T21DHS),
            ('3.0A', T30A),
            ('3.1A', T31A)
        ]

        tariff_cof = {
            '2.0A': 'A',
            '2.0DHA': 'B',
            '2.0DHS': 'D',
            '2.1A': 'A',
            '2.1DHA': 'B',
            '2.1DHS': 'D',
            '3.0A': 'C',
            '3.1A': 'C'
        }

        for t in tariffs:
            t_obj = get_tariff_by_code(t[0])()
            assert isinstance(t_obj, t[1])
            assert t_obj.cof, "Object doesn't have cof attribute"
            assert t_obj.cof == tariff_cof[t[0]], "Object cof not match with expected one"

    with it('must return None if the code is not in available'):
        t = get_tariff_by_code('NO_EXISTS')
        expect(t).to(be_none)
<commit_msg>Changed tests for NotNormPow and added tests for 6.1<commit_after>from expects.testing import failure
from expects import *
from enerdata.contracts.tariff import *
from datetime import datetime

with description('Create a period'):
    with it('accepts "te"'):
        TariffPeriod('P1', 'te')
    with it('accepts "tp"'):
        TariffPeriod('P1', 'tp')
    with it('fails when is not "te" nor "tp"'):
        with failure:
            TariffPeriod('P1', 'foo')
    with it('should raise an exception if range of hours is invalid'):
        expect(lambda: TariffPeriod('P1', 'te', winter_hours=[
            (0, 12), (11, 23)
        ])).to(raise_error(ValueError, 'Invalid winter hours'))
        expect(lambda: TariffPeriod('P1', 'te', summer_hours=[
            (0, 12), (11, 23)
        ])).to(raise_error(ValueError, 'Invalid summer hours'))

with description('A period'):
    with it('should have a range of hours 24 for winter by default'):
        p1 = TariffPeriod('P1', 'te')
        assert p1.winter_hours == [(0, 24)]
    with it('should be possible to set the range of winter hours in creation'):
        p1 = TariffPeriod('P1', 'te', winter_hours=[(0, 12)])
        assert p1.winter_hours == [(0, 12)]
    with it('should have a range of hours 24 for summer by default'):
        p1 = TariffPeriod('P1', 'te')
        assert p1.summer_hours == [(0, 24)]
    with it('should be possible to set the range of sumer hours in creation'):
        p1 = TariffPeriod('P1', 'te', summer_hours=[(0, 12)])
        assert p1.summer_hours == [(0, 12)]
    with it('has to validate range hours is correct'):
        assert check_range_hours([(0, 12)]) is True
        assert check_range_hours([(0, 12), (12, 24)]) is True
        assert check_range_hours([(-1, 0)]) is False
        assert check_range_hours([(0, 25)]) is False
        assert check_range_hours([(0, 0)]) is False
        assert check_range_hours([(4, 1)]) is False
        assert check_range_hours([(0, 1), (0, 2)]) is False
        assert check_range_hours([(0, 12), (12, 24)]) is True
    with it('should know the total of hours in summer'):
        p1 = TariffPeriod('P1', 'te', summer_hours=[(0, 12), (22, 24)])
        assert p1.total_summer_hours == 14
    with it('should know the total of hours in winter'):
        p1 = TariffPeriod('P1', 'te', winter_hours=[(12, 22)])
        assert p1.total_winter_hours == 10

with context('A tariff'):
    with before.all:
        self.tariff = Tariff('T1')
    with it('periods should be a tuple type'):
        assert isinstance(self.tariff.periods, tuple)
    with it('should return the number of periods of te'):
        self.tariff.periods = (
            TariffPeriod('1', 'te', winter_hours=[(12, 22)], summer_hours=[(13, 23)]),
            TariffPeriod('2', 'te', winter_hours=[(0, 12), (22, 24)], summer_hours=[(0, 13), (23, 24)])
        )
        assert self.tariff.get_number_of_periods() == 2
    with it('should return the periods of energy'):
        assert len(self.tariff.energy_periods) == 2
        assert self.tariff.energy_periods.keys() == ['1', '2']
    with it('should return the periods of power'):
        assert len(self.tariff.power_periods) == 0
        self.tariff.periods += (TariffPeriod('1', 'tp'),)
        assert len(self.tariff.power_periods) == 1
        assert self.tariff.power_periods.keys() == ['1']
    with it('should have 24h of range ours in its energy periods'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('1', 'te', summer_hours=[(12, 22)]),
                TariffPeriod('2', 'te', summer_hours=[(0, 12), (22, 23)])
            )
        expect(set_periods).to(raise_error(ValueError))

    with it('should check range of hours'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('1', 'te', summer_hours=[(13, 23)]),
                TariffPeriod('2', 'te', summer_hours=[(0, 12), (22, 24)])
            )
        expect(set_periods).to(raise_error(ValueError))

    with it('should check range and hours if a holiday period is defined'):
        def set_periods():
            self.tariff.periods = (
                TariffPeriod('P1', 'te', winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
                TariffPeriod('P2', 'te', winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
                TariffPeriod('P3', 'te', winter_hours=[(0, 8)], summer_hours=[(0, 8)]),
                TariffPeriod('P4', 'te', holiday=True, winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
                TariffPeriod('P5', 'te', holiday=True, winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
                TariffPeriod('P6', 'te', holiday=True, winter_hours=[(0, 8)], summer_hours=[(1, 8)])
            )
        expect(set_periods).to(raise_error(ValueError, 'The sum of hours in summer (in holidays) must be 24h: [(1, 8), (8, 11), (11, 15), (15, 24)]'))

    with it('should find the period by datetime'):
        self.tariff.periods = (
            TariffPeriod('P1', 'te', winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
            TariffPeriod('P2', 'te', winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
            TariffPeriod('P3', 'te', winter_hours=[(0, 8)], summer_hours=[(0, 8)]),
            TariffPeriod('P4', 'te', holiday=True, winter_hours=[(18, 22)], summer_hours=[(11, 15)]),
            TariffPeriod('P5', 'te', holiday=True, winter_hours=[(8, 18), (22, 24)], summer_hours=[(8, 11), (15, 24)]),
            TariffPeriod('P6', 'te', holiday=True, winter_hours=[(0, 8)], summer_hours=[(0, 8)])
        )
        dt = datetime(2015, 12, 24, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P1'
        dt = datetime(2015, 12, 25, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 19, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P4'
        dt = datetime(2015, 12, 27, 17, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P5'
        dt = datetime(2015, 12, 27, 1, 0, 0)
        period = self.tariff.get_period_by_date(dt)
        assert period.code == 'P6'
    with it('should allow to check if a set of powers is correct'):
        tari_T20A = T20A()
        expect(lambda: tari_T20A.evaluate_powers([-10])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T20A.evaluate_powers([0])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T20A.evaluate_powers([5])).to(
            raise_error(NotNormalizedPower))
        assert tari_T20A.evaluate_powers([5.5])
        expect(lambda: tari_T20A.evaluate_powers([5, 7])).to(
            raise_error(IncorrectPowerNumber, 'Expected 1 power(s) and got 2'))
        expect(lambda: tari_T20A.evaluate_powers([100])).to(
            raise_error(IncorrectMaxPower))

        tari_T30A = T30A()
        expect(lambda: tari_T30A.evaluate_powers([-10, -5, 0])).to(
            raise_error(NotPositivePower))
        expect(lambda: tari_T30A.evaluate_powers([10, 13, 15])).to(
            raise_error(IncorrectMaxPower))
        expect(lambda: tari_T30A.evaluate_powers([10, 13, 16])).to(
            raise_error(NotNormalizedPower))
        assert tari_T30A.evaluate_powers([2.304, 2.304, 16.454])
        expect(lambda: tari_T30A.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 3 power(s) and got 2'))

        tari_T31A = T31A()
        expect(lambda: tari_T31A.evaluate_powers([-10, -5, 0])).to(
            raise_error(NotPositivePower))
        assert tari_T31A.evaluate_powers([10, 13, 16])
        expect(lambda: tari_T31A.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 3 power(s) and got 2'))
        expect(lambda: tari_T31A.evaluate_powers([16, 20, 16])).to(
            raise_error(NotAscendingPowers))

        tari_T61A = T61A()
        expect(lambda: tari_T61A.evaluate_powers([-10, -5, 0, 10, 20])).to(
            raise_error(NotPositivePower))
        assert tari_T61A.evaluate_powers([400, 410, 420, 430, 440, 451])
        assert tari_T61A.evaluate_powers([500, 600, 700, 800, 900, 1000])
        expect(lambda: tari_T61A.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 6 power(s) and got 2'))
        expect(
            lambda: tari_T61A.evaluate_powers([500, 600, 700, 700, 600, 500])
        ).to(
            raise_error(NotAscendingPowers))

        tari_T61B = T61B()
        expect(lambda: tari_T61B.evaluate_powers([-10, -5, 0, 10, 20])).to(
            raise_error(NotPositivePower))
        assert tari_T61B.evaluate_powers([400, 410, 420, 430, 440, 451])
        assert tari_T61B.evaluate_powers([500, 600, 700, 800, 900, 1000])
        expect(lambda: tari_T61B.evaluate_powers([16, 17])).to(
            raise_error(IncorrectPowerNumber, 'Expected 6 power(s) and got 2'))
        expect(
            lambda: tari_T61B.evaluate_powers([500, 600, 700, 700, 600, 500])
        ).to(
            raise_error(NotAscendingPowers))
    with it('should allow to check if a maximum power is correct'):
        tari_T20A = T20A()
        assert not tari_T20A.is_maximum_power_correct(-10)
        assert not tari_T20A.is_maximum_power_correct(0)
        assert tari_T20A.is_maximum_power_correct(7)
        assert tari_T20A.is_maximum_power_correct(10)
        assert not tari_T20A.is_maximum_power_correct(1000)

with context('3.0A tariff'):
    with before.all:
        self.tariff = T30A()
        self.periods = self.tariff.energy_periods
    with it("should return 6 energy periods"):
        assert len(self.periods) == 6
    with it("should contain P1 to P6 periods"):
        assert 'P1' in self.periods.keys()
        assert 'P2' in self.periods.keys()
        assert 'P3' in self.periods.keys()
        assert 'P4' in self.periods.keys()
        assert 'P5' in self.periods.keys()
        assert 'P6' in self.periods.keys()

with description('Getting a tariff by descripion'):

    with it('must return the appropiate tariff and initialize the expected cof'):
        tariffs = [
            ('2.0A', T20A),
            ('2.0DHA', T20DHA),
            ('2.0DHS', T20DHS),
            ('2.1A', T21A),
            ('2.1DHA', T21DHA),
            ('2.1DHS', T21DHS),
            ('3.0A', T30A),
            ('3.1A', T31A)
        ]

        tariff_cof = {
            '2.0A': 'A',
            '2.0DHA': 'B',
            '2.0DHS': 'D',
            '2.1A': 'A',
            '2.1DHA': 'B',
            '2.1DHS': 'D',
            '3.0A': 'C',
            '3.1A': 'C'
        }

        for t in tariffs:
            t_obj = get_tariff_by_code(t[0])()
            assert isinstance(t_obj, t[1])
            assert t_obj.cof, "Object doesn't have cof attribute"
            assert t_obj.cof == tariff_cof[t[0]], "Object cof not match with expected one"

    with it('must return None if the code is not in available'):
        t = get_tariff_by_code('NO_EXISTS')
        expect(t).to(be_none)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
from datetime import datetime
import time
from dateutil import rrule

from flask import g, current_app, Blueprint, render_template, request, flash, Response, json
from flask.ext.login import login_required, current_user
from flask.ext.babel import gettext as _

from .models import Station, Program, ScheduledBlock, ScheduledProgram, Location, Person
from .forms import StationForm, ProgramForm, BlockForm, LocationForm, ScheduleProgramForm, PersonForm

from ..decorators import returns_json
from ..utils import error_dict
from ..extensions import db

radio = Blueprint('radio', __name__, url_prefix='/radio')

@radio.route('/', methods=['GET'])
def index():
    stations = Station.query.all()
    return render_template('radio/index.html',stations=stations)

@radio.route('/emergency/', methods=['GET'])
def emergency():
    stations = Station.query.all()
    #demo, override station statuses
    for s in stations:
        s.status = "on"

    #end demo
    return render_template('radio/emergency.html',stations=stations)

@radio.route('/station/', methods=['GET'])
def stations():
    stations = Station.query.all()
    return render_template('radio/stations.html', stations=stations, active='stations')

@radio.route('/station/<int:station_id>', methods=['GET', 'POST'])
def station(station_id):
    station = Station.query.filter_by(id=station_id).first_or_404()
    form = StationForm(obj=station, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(station)

        db.session.add(station)
        db.session.commit()
        flash(_('Station updated.'), 'success')

    return render_template('radio/station.html', station=station, form=form)

@radio.route('/station/add/', methods=['GET', 'POST'])
@login_required
def station_add():
    form = StationForm(request.form)
    station = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        cleaned_data.pop('phone_inline',None) #and also inline forms
        cleaned_data.pop('location_inline',None)
        station = Station(**cleaned_data) #create new object from data

        db.session.add(station)
        db.session.commit()
        flash(_('Station added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/station.html', station=station, form=form)

@radio.route('/program/', methods=['GET'])
def programs():
    programs = Program.query.all()
    return render_template('radio/programs.html', programs=programs, active='programs')

@radio.route('/program/<int:program_id>', methods=['GET', 'POST'])
def program(program_id):
    program = Program.query.filter_by(id=program_id).first_or_404()
    form = ProgramForm(obj=program, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(program)

        db.session.add(program)
        db.session.commit()
        flash(_('Program updated.'), 'success')

    return render_template('radio/program.html', program=program, form=form)

@radio.route('/program/add/', methods=['GET', 'POST'])
@login_required
def program_add():
    form = ProgramForm(request.form)
    program = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        program = Program(**cleaned_data) #create new object from data

        db.session.add(program)
        db.session.commit()
        flash(_('Program added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/program.html', program=program, form=form)

@radio.route('/people/', methods=['GET'])
def people():
    people = Person.query.all()
    return render_template('radio/people.html', people=people, active='people')

@radio.route('/people/<int:person_id>', methods=['GET', 'POST'])
def person(person_id):
    person = Person.query.filter_by(id=person_id).first_or_404()
    form = PersonForm(obj=person, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(person)

        db.session.add(person)
        db.session.commit()
        flash(_('Person updated.'), 'success')

    return render_template('radio/person.html', person=person, form=form)

@radio.route('/people/add/', methods=['GET', 'POST'])
@login_required
def person_add():
    form = PersonForm(request.form)
    person = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        person = Person(**cleaned_data) #create new object from data

        db.session.add(person)
        db.session.commit()
        flash(_('Person added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/person.html', person=person, form=form)

@radio.route('/location/add/inline/', methods=['POST'])
@login_required
@returns_json
def location_add_inline():
    data = json.loads(request.data)
    #handle floats individually
    float_vals = ['latitude','longitude']
    for field in float_vals:
        try:
            data[field] = float(data[field])
        except ValueError:
            response = {'status':'error','errors':{field:_('Invalid ')+field},'status_code':400}
            return response

    form = LocationForm(None, **data) #use this format to avoid multidict-type issue
    location = None
    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        location = Location(**cleaned_data) #create new object from data
        db.session.add(location)
        db.session.commit()
        response = {'status':'success','result':{'id':location.id,'string':unicode(location)},'status_code':200}
    elif request.method == "POST":
        #convert the error dictionary to something serializable
        response = {'status':'error','errors':error_dict(form.errors),'status_code':400}
    return response

@radio.route('/block/', methods=['GET'])
def scheduled_blocks():
    scheduled_blocks = ScheduledBlock.query.all()
    #TODO, display only those that are scheduled on stations the user can view

    return render_template('radio/scheduled_blocks.html', scheduled_blocks=scheduled_blocks, active='blocks')

@radio.route('/block/<int:block_id>', methods=['GET', 'POST'])
def scheduled_block(block_id):
    block = ScheduledBlock.query.filter_by(id=block_id).first_or_404()
    form = BlockForm(obj=block, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(block)
        db.session.add(block)
        db.session.commit()
        flash(_('Block updated.'), 'success')

    return render_template('radio/scheduled_block.html', scheduled_block=block, form=form)

@radio.route('/block/add/', methods=['GET', 'POST'])
@login_required
def scheduled_block_add():
    form = BlockForm(request.form)
    block = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        block = ScheduledBlock(**cleaned_data) #create new object from data

        db.session.add(block)
        db.session.commit()
        flash(_('Block added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/scheduled_block.html', block=block, form=form)

@radio.route('/scheduleprogram/add/inline/', methods=['POST'])
@login_required
@returns_json
def schedule_program_inline():
    data = json.loads(request.data)

    form = ScheduleProgramForm(None, **data)
    current_app.logger.debug( "*** form",form.data)
    #lookup fk's manually?

    scheduled_episode = None
    if form.validate_on_submit():
    #    cleaned_data = form.data #make a copy
    #    cleaned_data.pop('submit',None) #remove submit field from list
        scheduled_episode = ScheduledEpisode(**cleaned_data) #create new object from data
        db.session.add(scheduled_episode)
        db.session.commit()
        response = {'status':'success','result':{'id':scheduled_episode.id,'string':unicode(scheduled_episode)},'status_code':200}
    elif request.method == "POST":
        response = {'status':'error','errors':error_dict(form.errors),'status_code':400}
    return response

@radio.route('/station/<int:station_id>/scheduledepisodes.json', methods=['GET'])
@returns_json
def scheduled_episodes_json(station_id):
    scheduled_episodes = ScheduledEpisode.query.filter_by(station_id=station_id)
    resp = []
    for s in scheduled_episodes:
        current_app.logger.debug(s)
    return resp

@radio.route('/station/<int:station_id>/scheduledblocks.json', methods=['GET'])
@returns_json
def scheduled_block_json(station_id):
    scheduled_blocks = ScheduledBlock.query.filter_by(station_id=station_id)
    start = request.args.get('start')
    end = request.args.get('end')
    #TODO: hook fullcalendar updates into these params

    resp = []
    for block in scheduled_blocks:
        r = rrule.rrulestr(block.recurrence)
        for instance in r.between(start,end):
            d = {'title':block.name,
                'start':datetime.combine(instance,block.start_time),
                'end':datetime.combine(instance,block.end_time)}
            resp.append(d)
    return resp

@radio.route('/schedule/', methods=['GET'])
def schedule():
    #TODO, if user is authorized to view only one station, redirect them there

    stations = Station.query.all()
    return render_template('radio/schedules.html',
        stations=stations, active='schedule')

@radio.route('/schedule/<int:station_id>/', methods=['GET'])
def schedule_station(station_id):
    station = Station.query.get(station_id)

    scheduled_blocks = ScheduledBlock.query.filter_by(station_id=station.id)
    block_list = []
    for block in scheduled_blocks:
        r = rrule.rrulestr(block.recurrence)
        for instance in r[:10]: #TODO: dynamically determine instance limit from calendar view
            d = {'title':block.name,
                'start':datetime.combine(instance,block.start_time),
                'end':datetime.combine(instance,block.end_time)}
            block_list.append(d)

    form = ScheduleProgramForm()
    return render_template('radio/schedule.html',
        form=form, station=station, block_list=block_list,
        active='schedule')
<commit_msg>initial view for schedule_programs_inline<commit_after># -*- coding: utf-8 -*-

import os
from datetime import datetime
import time
from dateutil import rrule

from flask import g, current_app, Blueprint, render_template, request, flash, Response, json
from flask.ext.login import login_required, current_user
from flask.ext.babel import gettext as _

from .models import Station, Program, ScheduledBlock, ScheduledProgram, Location, Person
from .forms import StationForm, ProgramForm, BlockForm, LocationForm, ScheduleProgramForm, PersonForm

from ..decorators import returns_json
from ..utils import error_dict
from ..extensions import db

radio = Blueprint('radio', __name__, url_prefix='/radio')

@radio.route('/', methods=['GET'])
def index():
    stations = Station.query.all()
    return render_template('radio/index.html',stations=stations)

@radio.route('/emergency/', methods=['GET'])
def emergency():
    stations = Station.query.all()
    #demo, override station statuses
    for s in stations:
        s.status = "on"

    #end demo
    return render_template('radio/emergency.html',stations=stations)

@radio.route('/station/', methods=['GET'])
def stations():
    stations = Station.query.all()
    return render_template('radio/stations.html', stations=stations, active='stations')

@radio.route('/station/<int:station_id>', methods=['GET', 'POST'])
def station(station_id):
    station = Station.query.filter_by(id=station_id).first_or_404()
    form = StationForm(obj=station, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(station)

        db.session.add(station)
        db.session.commit()
        flash(_('Station updated.'), 'success')

    return render_template('radio/station.html', station=station, form=form)

@radio.route('/station/add/', methods=['GET', 'POST'])
@login_required
def station_add():
    form = StationForm(request.form)
    station = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        cleaned_data.pop('phone_inline',None) #and also inline forms
        cleaned_data.pop('location_inline',None)
        station = Station(**cleaned_data) #create new object from data

        db.session.add(station)
        db.session.commit()
        flash(_('Station added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/station.html', station=station, form=form)

@radio.route('/program/', methods=['GET'])
def programs():
    programs = Program.query.all()
    return render_template('radio/programs.html', programs=programs, active='programs')

@radio.route('/program/<int:program_id>', methods=['GET', 'POST'])
def program(program_id):
    program = Program.query.filter_by(id=program_id).first_or_404()
    form = ProgramForm(obj=program, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(program)

        db.session.add(program)
        db.session.commit()
        flash(_('Program updated.'), 'success')

    return render_template('radio/program.html', program=program, form=form)

@radio.route('/program/add/', methods=['GET', 'POST'])
@login_required
def program_add():
    form = ProgramForm(request.form)
    program = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        program = Program(**cleaned_data) #create new object from data

        db.session.add(program)
        db.session.commit()
        flash(_('Program added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/program.html', program=program, form=form)

@radio.route('/people/', methods=['GET'])
def people():
    people = Person.query.all()
    return render_template('radio/people.html', people=people, active='people')

@radio.route('/people/<int:person_id>', methods=['GET', 'POST'])
def person(person_id):
    person = Person.query.filter_by(id=person_id).first_or_404()
    form = PersonForm(obj=person, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(person)

        db.session.add(person)
        db.session.commit()
        flash(_('Person updated.'), 'success')

    return render_template('radio/person.html', person=person, form=form)

@radio.route('/people/add/', methods=['GET', 'POST'])
@login_required
def person_add():
    form = PersonForm(request.form)
    person = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        person = Person(**cleaned_data) #create new object from data

        db.session.add(person)
        db.session.commit()
        flash(_('Person added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/person.html', person=person, form=form)

@radio.route('/location/add/inline/', methods=['POST'])
@login_required
@returns_json
def location_add_inline():
    data = json.loads(request.data)
    #handle floats individually
    float_vals = ['latitude','longitude']
    for field in float_vals:
        try:
            data[field] = float(data[field])
        except ValueError:
            response = {'status':'error','errors':{field:_('Invalid ')+field},'status_code':400}
            return response

    form = LocationForm(None, **data) #use this format to avoid multidict-type issue
    location = None
    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        location = Location(**cleaned_data) #create new object from data
        db.session.add(location)
        db.session.commit()
        response = {'status':'success','result':{'id':location.id,'string':unicode(location)},'status_code':200}
    elif request.method == "POST":
        #convert the error dictionary to something serializable
        response = {'status':'error','errors':error_dict(form.errors),'status_code':400}
    return response

@radio.route('/block/', methods=['GET'])
def scheduled_blocks():
    scheduled_blocks = ScheduledBlock.query.all()
    #TODO, display only those that are scheduled on stations the user can view

    return render_template('radio/scheduled_blocks.html', scheduled_blocks=scheduled_blocks, active='blocks')

@radio.route('/block/<int:block_id>', methods=['GET', 'POST'])
def scheduled_block(block_id):
    block = ScheduledBlock.query.filter_by(id=block_id).first_or_404()
    form = BlockForm(obj=block, next=request.args.get('next'))

    if form.validate_on_submit():
        form.populate_obj(block)
        db.session.add(block)
        db.session.commit()
        flash(_('Block updated.'), 'success')

    return render_template('radio/scheduled_block.html', scheduled_block=block, form=form)

@radio.route('/block/add/', methods=['GET', 'POST'])
@login_required
def scheduled_block_add():
    form = BlockForm(request.form)
    block = None

    if form.validate_on_submit():
        cleaned_data = form.data #make a copy
        cleaned_data.pop('submit',None) #remove submit field from list
        block = ScheduledBlock(**cleaned_data) #create new object from data

        db.session.add(block)
        db.session.commit()
        flash(_('Block added.'), 'success')
    elif request.method == "POST":
        flash(_('Validation error'),'error')

    return render_template('radio/scheduled_block.html', block=block, form=form)

@radio.route('/scheduleprogram/add/inline/', methods=['POST'])
@login_required
@returns_json
def schedule_program_inline():
    data = json.loads(request.data)

    #lookup fks manually, this seems really hack-ish
    #TODO: cleanup
    fk_lookup = {'program':Program,'station':Station}
    for (field,cls) in fk_lookup.items():
        try:
            data[field] = cls.query.get(int(data[field]))
        except ValueError:
            response = {'status':'error','errors':{field:_('Invalid ')+field},'status_code':400}
            return response

    form = ScheduleProgramForm(None, **data)

    try:
        air_time = time.strptime(form.data['air_time'],'%H:%M')
    except ValueError:
        response = {'status':'error','errors':{'air_time':'Invalid time'},'status_code':400}
        return response

    if form.validate_on_submit():
        #parse recurrence rule
        r = rrule.rrulestr(form.data['recurrence'])
        for instance in r[:10]: #TODO: dynamically determine instance limit
            scheduled_program = ScheduledProgram(program=form.data['program'],
                                                station=form.data['station'])
            scheduled_program.start = datetime.combine(instance,air_time),
            scheduled_program.end = scheduled_program.start + program.duration
            db.session.add(scheduled_program)

        db.session.commit()
        response = {'status':'success','result':{},'status_code':200}
    elif request.method == "POST":
        response = {'status':'error','errors':error_dict(form.errors),'status_code':400}
    return response

@radio.route('/station/<int:station_id>/scheduledprograms.json', methods=['GET'])
@returns_json
def scheduled_programs_json(station_id):
    if request.args.get('start') and request.args.get('end'):
        start = datetime.utcfromtimestamp(float(request.args.get('start')))
        end = datetime.utcfromtimestamp(float(request.args.get('end')))
        scheduled_programs = ScheduledProgram.query.filter_by(station_id=station_id)
        #TODO: filter by start > start, end < end
    else:
        scheduled_programs = ScheduledProgram.query.filter_by(station_id=station_id)
    resp = []
    for s in scheduled_programs:
        current_app.logger.debug(s)
    return resp

@radio.route('/station/<int:station_id>/scheduledblocks.json', methods=['GET'])
@returns_json
def scheduled_block_json(station_id):
    scheduled_blocks = ScheduledBlock.query.filter_by(station_id=station_id)
    start = datetime.utcfromtimestamp(float(request.args.get('start')))
    end = datetime.utcfromtimestamp(float(request.args.get('end')))
    #TODO: hook fullcalendar updates into these params

    resp = []
    for block in scheduled_blocks:
        r = rrule.rrulestr(block.recurrence)
        for instance in r.between(start,end):
            d = {'title':block.name,
                'start':datetime.combine(instance,block.start_time),
                'end':datetime.combine(instance,block.end_time)}
            resp.append(d)
    return resp

@radio.route('/schedule/', methods=['GET'])
def schedule():
    #TODO, if user is authorized to view only one station, redirect them there

    stations = Station.query.all()
    return render_template('radio/schedules.html',
        stations=stations, active='schedule')

@radio.route('/schedule/<int:station_id>/', methods=['GET'])
def schedule_station(station_id):
    station = Station.query.get(station_id)

    #TODO: move this logic to an ajax call, like scheduled_block_json
    scheduled_blocks = ScheduledBlock.query.filter_by(station_id=station.id)
    block_list = []
    for block in scheduled_blocks:
        r = rrule.rrulestr(block.recurrence)
        for instance in r[:10]: #TODO: dynamically determine instance limit from calendar view
            d = {'title':block.name,
                'start':datetime.combine(instance,block.start_time),
                'end':datetime.combine(instance,block.end_time)}
            block_list.append(d)

    form = ScheduleProgramForm()
    return render_template('radio/schedule.html',
        form=form, station=station, block_list=block_list,
        active='schedule')
<|endoftext|>


<commit_before>from collatex import *
from xml.dom import pulldom
import re
import glob
from datetime import datetime, date
# import pytz
# from tzlocal import get_localzone

# today = date.today()
# utc_dt = datetime(today, tzinfo=pytz.utc)
# dateTime = utc_dt.astimezone(get_localzone())
# strDateTime = str(dateTime)

now = datetime.utcnow()
nowStr = str(now)

regexWhitespace = re.compile(r'\s+')
regexNonWhitespace = re.compile(r'\S+')
regexEmptyTag = re.compile(r'/>$')
regexBlankLine = re.compile(r'\n{2,}')
regexLeadingBlankLine = re.compile(r'^\n')
regexPageBreak = re.compile(r'<pb.+?/>')
RE_MARKUP = re.compile(r'<.+?>')
RE_PARA = re.compile(r'<p\s.+?/>')
RE_MILESTONE = re.compile(r'<milestone.+?/>')
RE_AMP_NSB = re.compile(r'\S&')
RE_AMP_NSE = re.compile(r'&\S')
RE_AMP = re.compile(r'&')
RE_MDEL = re.compile(r'<mdel>.+?/</mdel>')
RE_LB = re.compile(r'<lb.+?/>')
# ebb: RE_MDEL = those pesky deletions of two letters or less that we want to normalize out of the collation, but preserve in the output.

# Element types: xml, div, head, p, hi, pb, note, lg, l; comment()
# Tags to ignore, with content to keep: xml, comment, anchor
# Structural elements: div, p, lg, l
# Inline elements (empty) retained in normalization: pb, milestone, xi:include
# Inline and block elements (with content) retained in normalization: note, hi, head, ab

# GIs fall into one three classes
# 2017-05-21 ebb: Due to trouble with pulldom parsing XML comments, I have converted these to comment elements,
# 2017-05-21 ebb: to be ignored during collation.
# 2017-05-30 ebb: Determined that comment elements cannot really be ignored when they have text nodes (the text is
# 2017-05-30 ebb: collated but the tags are not). Decision to make the comments into self-closing elements with text
# 2017-05-30 ebb: contents as attribute values, and content such as tags simplified to be legal attribute values.
# 2017-05-22 ebb: I've set anchor elements with @xml:ids to be the indicators of collation "chunks" to process together
ignore = ['sourceDoc', 'xml', 'pb', 'comment', 'w', 'mod', 'anchor', 'include', 'delSpan', 'addSpan', 'add', 'handShift', 'damage', 'restore', 'zone', 'surface', 'graphic', 'unclear', 'retrace', 'damage', 'restore', 'hi', 'head', 'header']
inlineEmpty = ['lb', 'gap', 'del', 'p', 'div', 'milestone']
# 2018-05-12 ebb: I'm setting a white space on either side of the inlineEmpty elements in line 76
inlineContent = ['metamark', 'mdel']
blockElement = ['lg', 'l', 'note', 'ab', 'cit', 'quote', 'bibl']
# ebb: Tried removing 'comment', from blockElement list above, because we don't want these to be collated.

# 10-23-2017 ebb rv:

def normalizeSpace(inText):
    """Replaces all whitespace spans with single space characters"""
    if regexNonWhitespace.search(inText):
        return regexWhitespace.sub('\n', inText)
    else:
        return ''

def extract(input_xml):
    """Process entire input XML document, firing on events"""
    # Start pulling; it continues automatically
    doc = pulldom.parse(input_xml)
    output = ''
    for event, node in doc:
        # elements to ignore: xml
        if event == pulldom.START_ELEMENT and node.localName in ignore:
            continue
        # copy comments intact
        elif event == pulldom.COMMENT:
            doc.expandNode(node)
            output += node.toxml()
        # empty inline elements: pb, milestone
        elif event == pulldom.START_ELEMENT and node.localName in inlineEmpty:
            output += node.toxml()
        # non-empty inline elements: note, hi, head, l, lg, div, p, ab,
        elif event == pulldom.START_ELEMENT and node.localName in inlineContent:
            output += regexEmptyTag.sub('>', node.toxml())
        elif event == pulldom.END_ELEMENT and node.localName in inlineContent:
            output += '</' + node.localName + '>'
        elif event == pulldom.START_ELEMENT and node.localName in blockElement:
            output += '\n<' + node.localName + '>\n'
        elif event == pulldom.END_ELEMENT and node.localName in blockElement:
            output += '\n</' + node.localName + '>'
        elif event == pulldom.CHARACTERS:
            output += normalizeSpace(node.data)
        else:
            continue
    return output

def normalize(inputText):
   return RE_AMP.sub('and',\
        RE_MDEL.sub('', \
        RE_LB.sub('', \
        RE_MARKUP.sub('', inputText)))).lower()
#    return regexPageBreak('',inputText)

def processToken(inputText):
    return {"t": inputText + ' ', "n": normalize(inputText)}

def processWitness(inputWitness, id):
    return {'id': id, 'tokens': [processToken(token) for token in inputWitness]}

for name in glob.glob('../collationChunks/1818_fullFlat_*'):
    try:
        matchString = name.split("fullFlat_", 1)[1]
        # ebb: above gets C30.xml for example
        matchStr = matchString.split(".", 1)[0]
        # ebb: above strips off the file extension
        with open(name, 'rb') as f1818file, \
                open('../collationChunks/Thomas_fullFlat_' + matchString, 'rb') as fThomasfile, \
                open('../collationChunks/1823_fullFlat_' + matchString, 'rb') as f1823file, \
                open('../collationChunks/1831_fullFlat_' + matchString, 'rb') as f1831file, \
                open('../collationChunks/msColl_' + matchString, 'rb') as fMSfile, \
                open('../Full_xmlOutput/collation_' + matchStr + '.xml', 'w') as outputFile:
                # open('collationChunks/msColl_c56_' + matchString, 'rb') as fMSc56file, \
                # open('collationChunks/msColl_c58_' + matchString, 'rb') as fMSc58file, \
                # open('collationChunks/msColl_c57Frag_' + matchString, 'rb') as fMSc57Fragfile, \
                # open('collationChunks/msColl_c58Frag_' + matchString, 'rb') as fMSc58Fragfile, \
            # fMSc56_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc56file))).split('\n')
            # fMSc58_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc58file))).split('\n')
            # fMSc57Frag_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc57Fragfile))).split('\n')
            # fMSc58Frag_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc58Fragfile))).split('\n')
            f1818_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1818file))).split('\n')
            fThomas_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fThomasfile))).split('\n')
            f1823_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1823file))).split('\n')
            f1831_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1831file))).split('\n')
            fMS_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSfile))).split('\n')
            f1818_tokenlist = processWitness(f1818_tokens, 'f1818')
            fThomas_tokenlist = processWitness(fThomas_tokens, 'fThomas')
            f1823_tokenlist = processWitness(f1823_tokens, 'f1823')
            f1831_tokenlist = processWitness(f1831_tokens, 'f1831')
            fMS_tokenlist = processWitness(fMS_tokens, 'fMS')
            # fMSc56_tokenlist = processWitness(fMSc56_tokens, 'fMSc56')
            # fMSc58_tokenlist = processWitness(fMSc58_tokens, 'fMSc58')
            # fMSc57Frag_tokenlist = processWitness(fMSc57Frag_tokens, 'fMSc57Frag')
            # fMSc58Frag_tokenlist = processWitness(fMSc58Frag_tokens, 'fMSc58Frag')

            collation_input = {"witnesses": [f1818_tokenlist, fThomas_tokenlist, f1823_tokenlist, f1831_tokenlist, fMS_tokenlist]}
            # table = collate(collation_input, output='tei', segmentation=True)
            # table = collate(collation_input, segmentation=True, layout='vertical')
            table = collate(collation_input, output='xml', segmentation=True)
            print('<!-- ' + nowStr + ' -->' + table, file=outputFile)
            # print(table, file=outputFile)
    except IOError:
        pass

<commit_msg>revised normalize function and comment to screen out lb and mdel but keep for output<commit_after>from collatex import *
from xml.dom import pulldom
import re
import glob
from datetime import datetime, date
# import pytz
# from tzlocal import get_localzone

# today = date.today()
# utc_dt = datetime(today, tzinfo=pytz.utc)
# dateTime = utc_dt.astimezone(get_localzone())
# strDateTime = str(dateTime)

now = datetime.utcnow()
nowStr = str(now)

regexWhitespace = re.compile(r'\s+')
regexNonWhitespace = re.compile(r'\S+')
regexEmptyTag = re.compile(r'/>$')
regexBlankLine = re.compile(r'\n{2,}')
regexLeadingBlankLine = re.compile(r'^\n')
regexPageBreak = re.compile(r'<pb.+?/>')
RE_MARKUP = re.compile(r'<.+?>')
RE_PARA = re.compile(r'<p\s.+?/>')
RE_MILESTONE = re.compile(r'<milestone.+?/>')
RE_AMP_NSB = re.compile(r'\S&')
RE_AMP_NSE = re.compile(r'&\S')
RE_AMP = re.compile(r'&')
RE_MDEL = re.compile(r'<mdel>.+?/</mdel>')
RE_LB = re.compile(r'<lb.+?/>')
# ebb: RE_MDEL = those pesky deletions of two letters or less that we want to normalize out of the collation, but preserve in the output.

# Element types: xml, div, head, p, hi, pb, note, lg, l; comment()
# Tags to ignore, with content to keep: xml, comment, anchor
# Structural elements: div, p, lg, l
# Inline elements (empty) retained in normalization: pb, milestone, xi:include
# Inline and block elements (with content) retained in normalization: note, hi, head, ab

# GIs fall into one three classes
# 2017-05-21 ebb: Due to trouble with pulldom parsing XML comments, I have converted these to comment elements,
# 2017-05-21 ebb: to be ignored during collation.
# 2017-05-30 ebb: Determined that comment elements cannot really be ignored when they have text nodes (the text is
# 2017-05-30 ebb: collated but the tags are not). Decision to make the comments into self-closing elements with text
# 2017-05-30 ebb: contents as attribute values, and content such as tags simplified to be legal attribute values.
# 2017-05-22 ebb: I've set anchor elements with @xml:ids to be the indicators of collation "chunks" to process together
ignore = ['sourceDoc', 'xml', 'pb', 'comment', 'w', 'mod', 'anchor', 'include', 'delSpan', 'addSpan', 'add', 'handShift', 'damage', 'restore', 'zone', 'surface', 'graphic', 'unclear', 'retrace', 'damage', 'restore', 'hi', 'head', 'header']
inlineEmpty = ['lb', 'gap', 'del', 'p', 'div', 'milestone']
# 2018-05-12 ebb: I'm setting a white space on either side of the inlineEmpty elements in line 76
inlineContent = ['metamark', 'mdel']
blockElement = ['lg', 'l', 'note', 'ab', 'cit', 'quote', 'bibl']
# ebb: Tried removing 'comment', from blockElement list above, because we don't want these to be collated.

# 10-23-2017 ebb rv:

def normalizeSpace(inText):
    """Replaces all whitespace spans with single space characters"""
    if regexNonWhitespace.search(inText):
        return regexWhitespace.sub('\n', inText)
    else:
        return ''

def extract(input_xml):
    """Process entire input XML document, firing on events"""
    # Start pulling; it continues automatically
    doc = pulldom.parse(input_xml)
    output = ''
    for event, node in doc:
        # elements to ignore: xml
        if event == pulldom.START_ELEMENT and node.localName in ignore:
            continue
        # copy comments intact
        elif event == pulldom.COMMENT:
            doc.expandNode(node)
            output += node.toxml()
        # empty inline elements: pb, milestone
        elif event == pulldom.START_ELEMENT and node.localName in inlineEmpty:
            output += node.toxml()
        # non-empty inline elements: note, hi, head, l, lg, div, p, ab,
        elif event == pulldom.START_ELEMENT and node.localName in inlineContent:
            output += regexEmptyTag.sub('>', node.toxml())
        elif event == pulldom.END_ELEMENT and node.localName in inlineContent:
            output += '</' + node.localName + '>'
        elif event == pulldom.START_ELEMENT and node.localName in blockElement:
            output += '\n<' + node.localName + '>\n'
        elif event == pulldom.END_ELEMENT and node.localName in blockElement:
            output += '\n</' + node.localName + '>'
        elif event == pulldom.CHARACTERS:
            output += normalizeSpace(node.data)
        else:
            continue
    return output

def normalize(inputText):
   return RE_AMP.sub('and',\
        RE_MDEL.sub('', \
        RE_LB.sub('', \
        RE_MARKUP.sub('', inputText)))).lower()
#    return regexPageBreak('',inputText)
# ebb: The normalize function makes it possible to return normalized tokens that screen out some markup, but not all.

def processToken(inputText):
    return {"t": inputText + ' ', "n": normalize(inputText)}

def processWitness(inputWitness, id):
    return {'id': id, 'tokens': [processToken(token) for token in inputWitness]}

for name in glob.glob('../collationChunks/1818_fullFlat_*'):
    try:
        matchString = name.split("fullFlat_", 1)[1]
        # ebb: above gets C30.xml for example
        matchStr = matchString.split(".", 1)[0]
        # ebb: above strips off the file extension
        with open(name, 'rb') as f1818file, \
                open('../collationChunks/Thomas_fullFlat_' + matchString, 'rb') as fThomasfile, \
                open('../collationChunks/1823_fullFlat_' + matchString, 'rb') as f1823file, \
                open('../collationChunks/1831_fullFlat_' + matchString, 'rb') as f1831file, \
                open('../collationChunks/msColl_' + matchString, 'rb') as fMSfile, \
                open('../Full_xmlOutput/collation_' + matchStr + '.xml', 'w') as outputFile:
                # open('collationChunks/msColl_c56_' + matchString, 'rb') as fMSc56file, \
                # open('collationChunks/msColl_c58_' + matchString, 'rb') as fMSc58file, \
                # open('collationChunks/msColl_c57Frag_' + matchString, 'rb') as fMSc57Fragfile, \
                # open('collationChunks/msColl_c58Frag_' + matchString, 'rb') as fMSc58Fragfile, \
            # fMSc56_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc56file))).split('\n')
            # fMSc58_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc58file))).split('\n')
            # fMSc57Frag_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc57Fragfile))).split('\n')
            # fMSc58Frag_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSc58Fragfile))).split('\n')
            f1818_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1818file))).split('\n')
            fThomas_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fThomasfile))).split('\n')
            f1823_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1823file))).split('\n')
            f1831_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(f1831file))).split('\n')
            fMS_tokens = regexLeadingBlankLine.sub('', regexBlankLine.sub('\n', extract(fMSfile))).split('\n')
            f1818_tokenlist = processWitness(f1818_tokens, 'f1818')
            fThomas_tokenlist = processWitness(fThomas_tokens, 'fThomas')
            f1823_tokenlist = processWitness(f1823_tokens, 'f1823')
            f1831_tokenlist = processWitness(f1831_tokens, 'f1831')
            fMS_tokenlist = processWitness(fMS_tokens, 'fMS')
            # fMSc56_tokenlist = processWitness(fMSc56_tokens, 'fMSc56')
            # fMSc58_tokenlist = processWitness(fMSc58_tokens, 'fMSc58')
            # fMSc57Frag_tokenlist = processWitness(fMSc57Frag_tokens, 'fMSc57Frag')
            # fMSc58Frag_tokenlist = processWitness(fMSc58Frag_tokens, 'fMSc58Frag')

            collation_input = {"witnesses": [f1818_tokenlist, fThomas_tokenlist, f1823_tokenlist, f1831_tokenlist, fMS_tokenlist]}
            # table = collate(collation_input, output='tei', segmentation=True)
            # table = collate(collation_input, segmentation=True, layout='vertical')
            table = collate(collation_input, output='xml', segmentation=True)
            print('<!-- ' + nowStr + ' -->' + table, file=outputFile)
            # print(table, file=outputFile)
    except IOError:
        pass

<|endoftext|>


<commit_before>from __future__ import print_function
import os
from keras.layers import LSTM, Conv1D
from keras.layers.core import Dense
from keras.models import Sequential, load_model
from keras.optimizers import Nadam
from src.model_template import ModelTemplate
from src.utility import get_out_path, np3to2
import pytest

NN_TYPE = ['FFNN', 'LSTM', 'CNN']

class NnModel(ModelTemplate):
    """
    predict and evaluate from features amd label dict
    """
    def __init__(self, features, labels, terms, kinds=NN_TYPE[0], model_path=None):
        assert kinds in NN_TYPE
        self.n_data, self.n_feature, self.n_label = features['train'].shape
        super().__init__(features, labels, terms, feature_1dim=(kinds in NN_TYPE[:1]))
        if model_path is not None:
            assert os.path.exists(model_path)
            self.model = load_model(model_path)
        self.kinds = kinds
        self.history = None
        self.predicted_labels = None

    def layer_stack(self, topology_arr):
        """
        structs Neural Network Layer and display summary
        param: topology_arr: list
        """
        self.model = Sequential()
        self._characteristic_layer_construct(topology_arr)
        self.model.compile(
            loss='mse',
            optimizer=Nadam()
        )
        self.model.summary()

    def _characteristic_layer_construct(self, topology_arr):
        if self.kinds is NN_TYPE[0]:
            self.model.add(Dense(
                topology_arr[0],
                input_dim=self.n_feature * self.n_label,
                activation='relu'
            ))
            for n_unit in topology_arr[1:]:
                self.model.add(Dense(
                    n_unit,
                    activation='relu'
                ))
            self.model.add(Dense(self.n_label))
        elif self.kinds is NN_TYPE[1]:
            self.model.add(LSTM(
                topology_arr[0],
                input_dim=self.n_feature * self.n_label,
            ))
            for n_unit in topology_arr[1:]:
                self.model.add(LSTM(n_unit))
            self.model.add(LSTM(self.n_label))
        else:
            assert False

    def inference(self, nb_epoch=1000, verbose=0, save=True, batchsize=None):
        """
        fitting parameter and get inference values
        :param batchsize: int
        :param nb_epoch: int
        :param verbose: int
        :param save: Bool
        """
        if batchsize is None:
            batchsize = len(self.features['train'])
        print("inference...")
        self.history = self.model.fit(
            self.features['train'],
            self.labels['train'],
            batch_size=batchsize,
            nb_epoch=nb_epoch,
            verbose=verbose,
            validation_data=(self.features['valid'], self.labels['valid'])
        )
        self.predict_all(verbose=0)
        if save is True:
            self.model.save(
                get_out_path(
                    self.kinds + "_model.h5"
                ))
<commit_msg>CHANGE model parameter<commit_after>from __future__ import print_function
import os
from keras.layers import LSTM, Conv1D
from keras.layers.core import Dense
from keras.models import Sequential, load_model
from keras.optimizers import Nadam
from src.model_template import ModelTemplate
from src.utility import get_out_path, np3to2
import pytest

NN_TYPE = ['FFNN', 'LSTM', 'CNN']

class NnModel(ModelTemplate):
    """
    predict and evaluate from features amd label dict
    """
    def __init__(self, features, labels, terms, kinds=NN_TYPE[0], model_path=None):
        assert kinds in NN_TYPE
        self.n_data, self.n_feature, self.n_label = features['train'].shape
        super().__init__(features, labels, terms, feature_1dim=(kinds in NN_TYPE[:1]))
        if model_path is not None:
            assert os.path.exists(model_path)
            self.model = load_model(model_path)
        self.kinds = kinds
        self.history = None
        self.predicted_labels = None

    def setup_layer(self, topology_arr):
        """
        structs Neural Network Layer and display summary
        param: topology_arr: list
        """
        self.model = Sequential()
        self._characteristic_layer_construct(topology_arr)
        self.model.compile(
            loss='mse',
            optimizer=Nadam()
        )
        self.model.summary()

    def _characteristic_layer_construct(self, topology_arr):
        if self.kinds is NN_TYPE[0]:
            self.model.add(Dense(
                topology_arr[0],
                input_dim=self.n_feature * self.n_label,
                activation='relu'
            ))
            for n_unit in topology_arr[1:]:
                self.model.add(Dense(
                    n_unit,
                    activation='relu'
                ))
            self.model.add(Dense(self.n_label))
        elif self.kinds is NN_TYPE[1]:
            self.model.add(LSTM(
                topology_arr[0],
                input_dim=self.n_feature * self.n_label,
            ))
            for n_unit in topology_arr[1:]:
                self.model.add(LSTM(n_unit))
            self.model.add(LSTM(self.n_label))
        else:
            assert False

    def inference(self, nb_epoch=1000, verbose=0, batchsize=None, save=False):
        """
        fitting parameter and get inference values
        :param batchsize: int
        :param nb_epoch: int
        :param verbose: int
        :param save: Bool
        """
        if batchsize is None:
            batchsize = len(self.features['train'])
        print("inference...")
        self.history = self.model.fit(
            self.features['train'],
            self.labels['train'],
            batch_size=batchsize,
            nb_epoch=nb_epoch,
            verbose=verbose,
            validation_data=(self.features['valid'], self.labels['valid'])
        )
        self.predict_all(verbose=0)
        if save is True:
            self.model.save(
                get_out_path(
                    self.kinds + "_model.h5"
                ))
<|endoftext|>