<commit_before>#!/usr/bin/env python
#
# Copyright (C) 2013 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

import os, sys, json, subprocess, pipes
import collections
import dxpy
from dxpy.utils.printing import *
from dxpy.utils.resolver import *
from dxpy.cli.exec_io import *

def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
    '''
    :param jbor: a dict that is a valid job-based object reference
    :type jbor: dict
    :param job_outputs: a dict of finished local jobs to their output hashes
    :type job_outputs: :class:`collections.OrderedDict`
    :returns: the referenced value if present
    :raises: :exc:`Exception` if the job-based object reference cannot be resolved

    TODO: Support metadata references
    '''
    ref_job_id = get_job_from_jbor(jbor)
    ref_job_field = get_field_from_jbor(jbor)
    if is_localjob_id(ref_job_id):
        if job_outputs.get(ref_job_id) is None:
            if should_resolve:
                sys.exit('Error: Job ' + ref_job_id + ' not found in local finished jobs')
            else:
                return jbor
        if ref_job_field not in job_outputs[ref_job_id]:
            sys.exit('Error: Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field ' + ref_job_field + ' was not found in its output')
        return job_outputs[ref_job_id][ref_job_field]
    else:
        dxjob = dxpy.DXJob(ref_job_id)
        try:
            dxjob.wait_on_done()
        except Exception as e:
            sys.exit('Error: Could not wait for ' + ref_job_id + ' to finish: ' + str(e))
        job_desc = dxjob.describe()
        if ref_job_field not in job_desc['output']:
            sys.exit('Error: Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field ' + ref_job_field + ' was not found in its output')
        return job_desc['output'][ref_job_field]

def resolve_job_references(io_hash, job_outputs, should_resolve=True):
    '''
    :param io_hash: an input or output hash in which to resolve any job-based object references possible
    :type io_hash: dict
    :param job_outputs: a mapping of finished local jobs to their output hashes
    :type job_outputs: dict
    :param should_resolve: whether it is an error if a job-based object reference in *io_hash* cannot be resolved yet
    :type should_resolve: boolean

    Modifies *io_hash* in-place.
    '''
    q = []

    for field in io_hash:
        if is_job_ref(io_hash[field]):
            io_hash[field] = resolve_job_ref(io_hash[field], job_outputs, should_resolve)
        elif isinstance(io_hash[field], list) or isinstance(io_hash[field], dict):
            q.append(io_hash[field])

    while len(q) > 0:
        thing = q.pop()
        if isinstance(thing, list):
            for i in range(len(thing)):
                if is_job_ref(thing[i]):
                    thing[i] = resolve_job_ref(thing[i], job_outputs, should_resolve)
                elif isinstance(thing[i], list) or isinstance(thing[i], dict):
                    q.append(thing[i])
        else:
            for field in thing:
                if is_job_ref(thing[field]):
                    thing[field] = resolve_job_ref(thing[field], job_outputs, should_resolve)
                elif isinstance(thing[field], list) or isinstance(thing[field], dict):
                    q.append(thing[field])

def ensure_env_vars():
    for var in ['DX_BASHTEST_CODE_PATH',
                'DX_BASHTEST_RESOURCES_PATH',
                'DX_BASHTEST_JOB_HOMEDIRS']:
        if var not in os.environ:
            sys.exit('Error: Cannot run bash entry point locally if the environment variable ' + var + ' has not been set')

def queue_entry_point(function, input_hash):
    '''
    :param function: function to run
    :param input_hash: input to new job
    :returns: new local job ID

    This function should only be called by a locally running job, so
    all relevant DX_BASHTEST_* environment variables should be set.

    This function will set up the home directory for the job, add an
    entry in job_outputs.json, and append the job information to the
    job_queue.json file.  (Both files found in
    $DX_BASHTEST_JOB_HOMEDIRS.)
    '''
    ensure_env_vars()

    all_job_outputs_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json')

    with open(all_job_outputs_path, 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)
        job_id = 'localjob-' + str(len(all_job_outputs))

    with open(all_job_outputs_path, 'w') as fd:
        all_job_outputs[job_id] = None
        json.dump(all_job_outputs, fd, indent=4)
        fd.write('\n')

    job_homedir = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)
    os.mkdir(job_homedir)

    job_queue_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_queue.json')
    with open(job_queue_path, 'r') as fd:
        job_queue = json.load(fd)
    job_queue.append({"id": job_id,
                      "function": function,
                      "input_hash": input_hash})
    with open(job_queue_path, 'w') as fd:
        json.dump(job_queue, fd, indent=4)
        fd.write('\n')

    return job_id

def run_one_entry_point(job_id, function, input_hash):
    '''
    :param job_id: job ID of the local job to run
    :param function: function to run
    :param input_hash: input for the job (may include job-based object references)

    Runs the specified entry point and retrieves the job's output,
    updating job_outputs.json (in $DX_BASHTEST_JOB_HOMEDIRS) appropriately.
    '''
    job_homedir = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)
    log_filenames = []

    watch = os.environ.get('DX_BASHTEST_WATCH') == '1'

    if os.environ.get('DX_BASHTEST_SPLIT_LOGS'):
        stdout_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '-stdout.log')
        log_filenames.append(stdout_path)
        job_stdout = open(stdout_path, 'w+')

        stderr_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '-stderr.log')
        log_filenames.append(stderr_path)
        job_stderr = open(stderr_path, 'w+')
    else:
        stdouterr_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '.log')
        log_filenames.append(stdouterr_path)
        job_stdout = open(stdouterr_path, 'w+')
        job_stderr = job_stdout

    job_env = os.environ.copy()
    job_env['HOME'] = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)

    all_job_outputs_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json')

    with open(all_job_outputs_path, 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)

    # Resolve local job-based object references
    resolve_job_references(input_hash, all_job_outputs)

    # Save job input to job_input.json
    with open(os.path.join(job_homedir, 'job_input.json'), 'w') as fd:
        json.dump(input_hash, fd, indent=4)
        fd.write('\n')

    # Save job input to env vars
    env_path = os.path.join(job_homedir, 'environment')
    with open(env_path, 'w') as fd:
        # Following code is what is used to generate env vars on the remote worker
        fd.write("\n".join(["export {k}=( {vlist} )".format(k=k, vlist=" ".join([pipes.quote(vitem if isinstance(vitem, basestring) else json.dumps(vitem)) for vitem in v])) if isinstance(v, list) else "export {k}={v}".format(k=k, v=pipes.quote(v if isinstance(v, basestring) else json.dumps(v))) for k, v in input_hash.iteritems()]))

    print job_id + ':' + function + ' -> ' + JOB_STATES('running')

    fn_process = subprocess.Popen(['bash', '-c', '-e'] + \
                                      (['-x'] if os.environ.get('DX_BASHTEST_X_FLAG') else []) + \
                                      ['cd {homedir}; . {env}; . {code}; if [[ $(type -t {function}) == "function" ]]; then {function}; else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2; fi'.format(homedir=job_homedir,
                                                                                env=os.path.join(job_env['HOME'], 'environment'),
                                                                                code=os.environ['DX_BASHTEST_CODE_PATH'],
                                                                                function=function)],
                                  stdout=job_stdout,
                                  stderr=job_stderr,
                                  env=job_env)

    fn_process.communicate()
    job_stderr.write('Exit code: ' + str(fn_process.returncode))

    if watch:
        print "Logs"
        print '-'*len("Logs")
        if os.environ.get('DX_BASHTEST_SPLIT_LOGS'):
            job_stdout.seek(0)
            print GREEN() + 'stdout:' + ENDC()
            for line in job_stdout:
                print '> ' + line.strip()
            job_stderr.seek(0)
            print YELLOW() + 'stderr:' + ENDC()
            for line in job_stderr:
                print '> ' + line.strip()
        else:
            job_stdout.seek(0)
            for line in job_stdout:
                print '> ' + line.strip()

    if fn_process.returncode != 0:
        sys.exit(fill(job_id + ':' + function + ' ' + JOB_STATES('failed') + ' (error code ' + str(fn_process.returncode) + ')') + ('' if watch else '\n' + fill('Consult the job\'s logs in:') + '\n  ' + '\n  '.join(log_filenames)))

    # Now updating job output aggregation file with job's output
    job_output_path = os.path.join(job_env['HOME'], 'job_output.json')
    if os.path.exists(job_output_path):
        try:
            with open(job_output_path, 'r') as fd:
                job_output = json.load(fd, object_pairs_hook=collections.OrderedDict)
        except BaseException as e:
            sys.exit(fill(str(e.__class__) + ': ' + str(e)))
    else:
        job_output = {}

    print job_id + ':' + function + ' -> ' + GREEN() + 'finished running' + ENDC()
    print job_output_to_str(job_output, prefix='  ')

    with open(os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)
    all_job_outputs[job_id] = job_output

    # Before dumping, see if any new jbors should be resolved now
    for other_job_id in all_job_outputs:
        if all_job_outputs[other_job_id] is None:
            # Skip if job is not done yet (true for ancestor jobs)
            continue
        resolve_job_references(all_job_outputs[other_job_id], all_job_outputs, should_resolve=False)

    with open(os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'w') as fd:
        json.dump(all_job_outputs, fd, indent=4)
        fd.write('\n')

def run_entry_points():
    '''
    Runs all job entry points found in
    $DX_BASHTEST_JOB_HOMEDIRS/job_queue.json in a first-in, first-out
    manner until it is an empty array (or an error occurs).
    '''
    job_queue_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_queue.json')
    while True:
        with open(job_queue_path, 'r') as fd:
            job_queue = json.load(fd)
        if len(job_queue) == 0:
            return
        entry_point_to_run = job_queue[0]
        with open(job_queue_path, 'w') as fd:
            json.dump(job_queue[1:], fd)
            fd.write('\n')
        run_one_entry_point(job_id=entry_point_to_run['id'],
                            function=entry_point_to_run['function'],
                            input_hash=entry_point_to_run['input_hash'])
<commit_msg>dx-jobutil-run-bash-app-locally now quotes its paths as well before passing them to bash<commit_after>#!/usr/bin/env python
#
# Copyright (C) 2013 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

import os, sys, json, subprocess, pipes
import collections
import dxpy
from dxpy.utils.printing import *
from dxpy.utils.resolver import *
from dxpy.cli.exec_io import *

def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
    '''
    :param jbor: a dict that is a valid job-based object reference
    :type jbor: dict
    :param job_outputs: a dict of finished local jobs to their output hashes
    :type job_outputs: :class:`collections.OrderedDict`
    :returns: the referenced value if present
    :raises: :exc:`Exception` if the job-based object reference cannot be resolved

    TODO: Support metadata references
    '''
    ref_job_id = get_job_from_jbor(jbor)
    ref_job_field = get_field_from_jbor(jbor)
    if is_localjob_id(ref_job_id):
        if job_outputs.get(ref_job_id) is None:
            if should_resolve:
                sys.exit('Error: Job ' + ref_job_id + ' not found in local finished jobs')
            else:
                return jbor
        if ref_job_field not in job_outputs[ref_job_id]:
            sys.exit('Error: Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field ' + ref_job_field + ' was not found in its output')
        return job_outputs[ref_job_id][ref_job_field]
    else:
        dxjob = dxpy.DXJob(ref_job_id)
        try:
            dxjob.wait_on_done()
        except Exception as e:
            sys.exit('Error: Could not wait for ' + ref_job_id + ' to finish: ' + str(e))
        job_desc = dxjob.describe()
        if ref_job_field not in job_desc['output']:
            sys.exit('Error: Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field ' + ref_job_field + ' was not found in its output')
        return job_desc['output'][ref_job_field]

def resolve_job_references(io_hash, job_outputs, should_resolve=True):
    '''
    :param io_hash: an input or output hash in which to resolve any job-based object references possible
    :type io_hash: dict
    :param job_outputs: a mapping of finished local jobs to their output hashes
    :type job_outputs: dict
    :param should_resolve: whether it is an error if a job-based object reference in *io_hash* cannot be resolved yet
    :type should_resolve: boolean

    Modifies *io_hash* in-place.
    '''
    q = []

    for field in io_hash:
        if is_job_ref(io_hash[field]):
            io_hash[field] = resolve_job_ref(io_hash[field], job_outputs, should_resolve)
        elif isinstance(io_hash[field], list) or isinstance(io_hash[field], dict):
            q.append(io_hash[field])

    while len(q) > 0:
        thing = q.pop()
        if isinstance(thing, list):
            for i in range(len(thing)):
                if is_job_ref(thing[i]):
                    thing[i] = resolve_job_ref(thing[i], job_outputs, should_resolve)
                elif isinstance(thing[i], list) or isinstance(thing[i], dict):
                    q.append(thing[i])
        else:
            for field in thing:
                if is_job_ref(thing[field]):
                    thing[field] = resolve_job_ref(thing[field], job_outputs, should_resolve)
                elif isinstance(thing[field], list) or isinstance(thing[field], dict):
                    q.append(thing[field])

def ensure_env_vars():
    for var in ['DX_BASHTEST_CODE_PATH',
                'DX_BASHTEST_RESOURCES_PATH',
                'DX_BASHTEST_JOB_HOMEDIRS']:
        if var not in os.environ:
            sys.exit('Error: Cannot run bash entry point locally if the environment variable ' + var + ' has not been set')

def queue_entry_point(function, input_hash):
    '''
    :param function: function to run
    :param input_hash: input to new job
    :returns: new local job ID

    This function should only be called by a locally running job, so
    all relevant DX_BASHTEST_* environment variables should be set.

    This function will set up the home directory for the job, add an
    entry in job_outputs.json, and append the job information to the
    job_queue.json file.  (Both files found in
    $DX_BASHTEST_JOB_HOMEDIRS.)
    '''
    ensure_env_vars()

    all_job_outputs_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json')

    with open(all_job_outputs_path, 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)
        job_id = 'localjob-' + str(len(all_job_outputs))

    with open(all_job_outputs_path, 'w') as fd:
        all_job_outputs[job_id] = None
        json.dump(all_job_outputs, fd, indent=4)
        fd.write('\n')

    job_homedir = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)
    os.mkdir(job_homedir)

    job_queue_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_queue.json')
    with open(job_queue_path, 'r') as fd:
        job_queue = json.load(fd)
    job_queue.append({"id": job_id,
                      "function": function,
                      "input_hash": input_hash})
    with open(job_queue_path, 'w') as fd:
        json.dump(job_queue, fd, indent=4)
        fd.write('\n')

    return job_id

def run_one_entry_point(job_id, function, input_hash):
    '''
    :param job_id: job ID of the local job to run
    :param function: function to run
    :param input_hash: input for the job (may include job-based object references)

    Runs the specified entry point and retrieves the job's output,
    updating job_outputs.json (in $DX_BASHTEST_JOB_HOMEDIRS) appropriately.
    '''
    job_homedir = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)
    log_filenames = []

    watch = os.environ.get('DX_BASHTEST_WATCH') == '1'

    if os.environ.get('DX_BASHTEST_SPLIT_LOGS'):
        stdout_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '-stdout.log')
        log_filenames.append(stdout_path)
        job_stdout = open(stdout_path, 'w+')

        stderr_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '-stderr.log')
        log_filenames.append(stderr_path)
        job_stderr = open(stderr_path, 'w+')
    else:
        stdouterr_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id + '.log')
        log_filenames.append(stdouterr_path)
        job_stdout = open(stdouterr_path, 'w+')
        job_stderr = job_stdout

    job_env = os.environ.copy()
    job_env['HOME'] = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], job_id)

    all_job_outputs_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json')

    with open(all_job_outputs_path, 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)

    # Resolve local job-based object references
    resolve_job_references(input_hash, all_job_outputs)

    # Save job input to job_input.json
    with open(os.path.join(job_homedir, 'job_input.json'), 'w') as fd:
        json.dump(input_hash, fd, indent=4)
        fd.write('\n')

    # Save job input to env vars
    env_path = os.path.join(job_homedir, 'environment')
    with open(env_path, 'w') as fd:
        # Following code is what is used to generate env vars on the remote worker
        fd.write("\n".join(["export {k}=( {vlist} )".format(k=k, vlist=" ".join([pipes.quote(vitem if isinstance(vitem, basestring) else json.dumps(vitem)) for vitem in v])) if isinstance(v, list) else "export {k}={v}".format(k=k, v=pipes.quote(v if isinstance(v, basestring) else json.dumps(v))) for k, v in input_hash.iteritems()]))

    print job_id + ':' + function + ' -> ' + JOB_STATES('running')

    fn_process = subprocess.Popen(['bash', '-c', '-e'] + \
                                      (['-x'] if os.environ.get('DX_BASHTEST_X_FLAG') else []) + \
                                      ['cd {homedir}; . {env}; . {code}; if [[ $(type -t {function}) == "function" ]]; then {function}; else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2; fi'.format(homedir=pipes.quote(job_homedir),
                                                                                env=pipes.quote(os.path.join(job_env['HOME'], 'environment')),
                                                                                code=pipes.quote(os.environ['DX_BASHTEST_CODE_PATH']),
                                                                                function=function)],
                                  stdout=job_stdout,
                                  stderr=job_stderr,
                                  env=job_env)

    fn_process.communicate()
    job_stderr.write('Exit code: ' + str(fn_process.returncode))

    if watch:
        print "Logs"
        print '-'*len("Logs")
        if os.environ.get('DX_BASHTEST_SPLIT_LOGS'):
            job_stdout.seek(0)
            print GREEN() + 'stdout:' + ENDC()
            for line in job_stdout:
                print '> ' + line.strip()
            job_stderr.seek(0)
            print YELLOW() + 'stderr:' + ENDC()
            for line in job_stderr:
                print '> ' + line.strip()
        else:
            job_stdout.seek(0)
            for line in job_stdout:
                print '> ' + line.strip()

    if fn_process.returncode != 0:
        sys.exit(fill(job_id + ':' + function + ' ' + JOB_STATES('failed') + ' (error code ' + str(fn_process.returncode) + ')') + ('' if watch else '\n' + fill('Consult the job\'s logs in:') + '\n  ' + '\n  '.join(log_filenames)))

    # Now updating job output aggregation file with job's output
    job_output_path = os.path.join(job_env['HOME'], 'job_output.json')
    if os.path.exists(job_output_path):
        try:
            with open(job_output_path, 'r') as fd:
                job_output = json.load(fd, object_pairs_hook=collections.OrderedDict)
        except BaseException as e:
            sys.exit(fill(str(e.__class__) + ': ' + str(e)))
    else:
        job_output = {}

    print job_id + ':' + function + ' -> ' + GREEN() + 'finished running' + ENDC()
    print job_output_to_str(job_output, prefix='  ')

    with open(os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)
    all_job_outputs[job_id] = job_output

    # Before dumping, see if any new jbors should be resolved now
    for other_job_id in all_job_outputs:
        if all_job_outputs[other_job_id] is None:
            # Skip if job is not done yet (true for ancestor jobs)
            continue
        resolve_job_references(all_job_outputs[other_job_id], all_job_outputs, should_resolve=False)

    with open(os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'w') as fd:
        json.dump(all_job_outputs, fd, indent=4)
        fd.write('\n')

def run_entry_points():
    '''
    Runs all job entry points found in
    $DX_BASHTEST_JOB_HOMEDIRS/job_queue.json in a first-in, first-out
    manner until it is an empty array (or an error occurs).
    '''
    job_queue_path = os.path.join(os.environ['DX_BASHTEST_JOB_HOMEDIRS'], 'job_queue.json')
    while True:
        with open(job_queue_path, 'r') as fd:
            job_queue = json.load(fd)
        if len(job_queue) == 0:
            return
        entry_point_to_run = job_queue[0]
        with open(job_queue_path, 'w') as fd:
            json.dump(job_queue[1:], fd)
            fd.write('\n')
        run_one_entry_point(job_id=entry_point_to_run['id'],
                            function=entry_point_to_run['function'],
                            input_hash=entry_point_to_run['input_hash'])
<|endoftext|>


<commit_before># BuildFox ninja generator

import os
import re
import sys
import shutil

re_folder_part = re.compile(r"^((?:\(\.\*\)\(\.\*\)|(?:[^\r\n(\[\"\\]|\\.))+)(\\\/|\/|\\).*$") # match folder part in filename regex
re_non_escaped_char = re.compile(r"(?<!\\)\\(.)") # looking for not escaped \ with char
re_capture_group_ref = re.compile(r"(?<!\\)\\(\d)") # match regex capture group reference

# return relative path to current work dir
def rel_dir(filename):
	path = os.path.relpath(os.path.dirname(os.path.abspath(filename)), os.getcwd()).replace("\\", "/") + "/"
	if path == "./":
		path = ""
	return path

# return regex value in filename for regex or wildcard
# replace_groups replace wildcards with group reference indexes
def wildcard_regex(filename, replace_groups = False):
	if filename.startswith("r\""):
		return filename[2:-1] # strip r" and "
	elif "*" in filename or "?" in filename or "[" in filename:
		# based on fnmatch.translate with each wildcard is a capture group
		i, n = 0, len(filename)
		groups = 1
		res = ""
		while i < n:
			c = filename[i]
			i = i + 1
			if c == "*":
				if replace_groups:
					res = res + "\\" + str(groups)
					groups += 1
				else:
					res = res + "(.*)"
			elif c == "?":
				if replace_groups:
					res = res + "\\" + str(groups)
					groups += 1
				else:
					res = res + "(.)"
			elif replace_groups:
				res = res + c
			elif c == "[":
				j = i
				if j < n and filename[j] == "!":
					j = j + 1
				if j < n and filename[j] == "]":
					j = j + 1
				while j < n and filename[j] != "]":
					j = j + 1
				if j >= n:
					res = res + "\\["
				else:
					stuff = filename[i:j].replace("\\", "\\\\")
					i = j + 1
					if stuff[0] == "!":
						stuff = "^" + stuff[1:]
					elif stuff[0] == "^":
						stuff = "\\" + stuff
					res = "%s([%s])" % (res, stuff)
			else:
				res = res + re.escape(c)
		if replace_groups:
			return res
		else:
			return res + "\Z(?ms)"
	else:
		return None

# input can be string or list of strings
# outputs are always lists
def find_files(inputs, outputs = None, rel_path = "", generated = None):
	# rename regex back to readable form
	def replace_non_esc(match_group):
		return match_group.group(1)
	if inputs:
		result = []
		matched = []
		for input in inputs:
			regex = wildcard_regex(input)
			print("----------------------")
			print("input %s" % input)
			print("regex %s" % regex)
			if regex:
				# find the folder where to look for files
				base_folder = re_folder_part.match(regex)
				if base_folder:
					print("matched %s" % str(base_folder))
					base_folder = base_folder.group(1) + base_folder.group(2)
					base_folder = re_non_escaped_char.sub(replace_non_esc, base_folder)
					print("basefolder %s" % str(base_folder))
					separator = "\\" if base_folder.rfind("\\") > base_folder.rfind("/") else "/"
					base_folder = os.path.dirname(base_folder)
					list_folder = rel_path + base_folder
					print("list_folder %s" % list_folder)
				else:
					separator = ""
					base_folder = ""
					if len(rel_path):
						list_folder = rel_path[:-1] # strip last /
					else:
						list_folder = "."

				# look for files
				list_folder = os.path.normpath(list_folder).replace("\\", "/")
				re_regex = re.compile(regex)
				if os.path.isdir(list_folder):
					fs_files = set(os.listdir(list_folder))
				else:
					fs_files = set()
				generated_files = generated.get(list_folder, set())
				# we must have stable sort here
				# so output ninja files will be same between runs
				all_files = sorted(list(fs_files.union(generated_files)))
				for file in all_files:
					name = base_folder + separator + file
					match = re_regex.match(name)
					if match:
						result.append(rel_path + name)
						matched.append(match.groups())
			else:
				result.append(rel_path + input)
		inputs = result

	if outputs:
		result = []
		for output in outputs:
			# we want \number instead of capture groups
			regex = wildcard_regex(output, True)
			if regex:
				for match in matched:
					# replace \number with data
					def replace_group(matchobj):
						index = int(matchobj.group(1)) - 1
						if index >= 0 and index < len(match):
							return match[index]
						else:
							return ""
					file = re_capture_group_ref.sub(replace_group, regex)
					file = re_non_escaped_char.sub(replace_non_esc, file)

					result.append(rel_path + file)
			else:
				result.append(rel_path + output)

		# normalize results
		result = [os.path.normpath(file).replace("\\", "/") for file in result]

	# normalize inputs
	inputs = [os.path.normpath(file).replace("\\", "/") for file in inputs]

	if outputs:
		return inputs, result
	else:
		return inputs

# finds the file in path
def which(cmd, mode = os.F_OK | os.X_OK, path = None):
	if sys.version_info[0:2] >= (3, 3):
		return shutil.which(cmd, mode, path)
	else:
		def _access_check(fn, mode):
			return (os.path.exists(fn) and os.access(fn, mode)
					and not os.path.isdir(fn))

		if os.path.dirname(cmd):
			if _access_check(cmd, mode):
				return cmd
			return None

		if path is None:
			path = os.environ.get("PATH", os.defpath)
		if not path:
			return None
		path = path.split(os.pathsep)

		if sys.platform == "win32":
			if not os.curdir in path:
				path.insert(0, os.curdir)
			pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
			if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
				files = [cmd]
			else:
				files = [cmd + ext for ext in pathext]
		else:
			files = [cmd]

		seen = set()
		for dir in path:
			normdir = os.path.normcase(dir)
			if not normdir in seen:
				seen.add(normdir)
				for thefile in files:
					name = os.path.join(dir, thefile)
					if _access_check(name, mode):
						return name
		return None
<commit_msg>rec glob wip<commit_after># BuildFox ninja generator

import os
import re
import sys
import shutil

re_folder_part = re.compile(r"^((?:\(\.\*\)\(\.\*\)|(?:[^\r\n(\[\"\\]|\\.))+)(\\\/|\/|\\).*$") # match folder part in filename regex
re_non_escaped_char = re.compile(r"(?<!\\)\\(.)") # looking for not escaped \ with char
re_capture_group_ref = re.compile(r"(?<!\\)\\(\d)") # match regex capture group reference

# return relative path to current work dir
def rel_dir(filename):
	path = os.path.relpath(os.path.dirname(os.path.abspath(filename)), os.getcwd()).replace("\\", "/") + "/"
	if path == "./":
		path = ""
	return path

# return regex value in filename for regex or wildcard
# replace_groups replace wildcards with group reference indexes
def wildcard_regex(filename, replace_groups = False):
	if filename.startswith("r\""):
		return filename[2:-1] # strip r" and "
	elif "*" in filename or "?" in filename or "[" in filename:
		# based on fnmatch.translate with each wildcard is a capture group
		i, n = 0, len(filename)
		groups = 1
		res = ""
		while i < n:
			c = filename[i]
			i = i + 1
			if c == "*":
				if replace_groups:
					res = res + "\\" + str(groups)
					groups += 1
				else:
					res = res + "(.*)"
			elif c == "?":
				if replace_groups:
					res = res + "\\" + str(groups)
					groups += 1
				else:
					res = res + "(.)"
			elif replace_groups:
				res = res + c
			elif c == "[":
				j = i
				if j < n and filename[j] == "!":
					j = j + 1
				if j < n and filename[j] == "]":
					j = j + 1
				while j < n and filename[j] != "]":
					j = j + 1
				if j >= n:
					res = res + "\\["
				else:
					stuff = filename[i:j].replace("\\", "\\\\")
					i = j + 1
					if stuff[0] == "!":
						stuff = "^" + stuff[1:]
					elif stuff[0] == "^":
						stuff = "\\" + stuff
					res = "%s([%s])" % (res, stuff)
			else:
				res = res + re.escape(c)
		if replace_groups:
			return res
		else:
			return res + "\Z(?ms)"
	else:
		return None

# input can be string or list of strings
# outputs are always lists
def find_files(inputs, outputs = None, rel_path = "", generated = None):
	# rename regex back to readable form
	def replace_non_esc(match_group):
		return match_group.group(1)
	if inputs:
		result = []
		matched = []
		for input in inputs:
			regex = wildcard_regex(input)
			print("----------------------")
			print("input %s" % input)
			print("regex %s" % regex)
			if regex:
				# find the folder where to look for files
				base_folder = re_folder_part.match(regex)
				if base_folder:
					print("matched %s" % str(base_folder))
					base_folder = base_folder.group(1) + base_folder.group(2)
					base_folder = re_non_escaped_char.sub(replace_non_esc, base_folder)
					print("basefolder %s" % str(base_folder))
					separator = "\\" if base_folder.rfind("\\") > base_folder.rfind("/") else "/"
					base_folder = os.path.dirname(base_folder)
					list_folder = rel_path + base_folder
					print("list_folder %s" % list_folder)
				else:
					separator = ""
					base_folder = ""
					if len(rel_path):
						list_folder = rel_path[:-1] # strip last /
					else:
						list_folder = "."

				# look for files
				list_folder = os.path.normpath(list_folder).replace("\\", "/")
				print("list_folder %s" % list_folder)

				all_listed_folders = list_folder.split("/")
				print("all_listed_folders:")
				from pprint import pprint
				pprint(all_listed_folders)

				is_recursive_glob = "(.*)(.*)" in all_listed_folders
				print("is_recursive_glob %s" % str(is_recursive_glob))

				fs_files = set()
				generated_files = set()

				if is_recursive_glob:
					tmp_folders = ["."]
					print(" --> TODO")

					for folder in all_listed_folders:
						if folder == "(.*)(.*)":
							print(folder)
							pprint(tmp_folders)
							tmp_folders2 = []
							for tmp_folder in tmp_folders:
								print(tmp_folder)
								for root, dirnames, filenames in os.walk(tmp_folder):
									for dir in dirnames:
										tmp_folders2.append(os.path.join(root, dir).replace("\\", "/"))
										print(root + " === " + dir)
							# TODO filtering ?
							tmp_folders = tmp_folders2

						else:
							tmp_folders = [f + separator + folder for f in tmp_folders]
					pprint(tmp_folders)
				else:
					if os.path.isdir(list_folder):
						fs_files = set(os.listdir(list_folder))
						fs_files = set([base_folder + separator + name for name in fs_files()])

					generated_files = generated.get(list_folder, set())
					generated_files = set([base_folder + separator + name for name in generated_files])

				# we must have stable sort here
				# so output ninja files will be same between runs
				all_files = list(fs_files.union(generated_files))
				all_files = sorted(all_files)

				# while capturing ** we want just to capture *
				if is_recursive_glob:
					regex = regex.replace("(.*)(.*)", "(.*)") # TODO check if this is correct

				print("final regex : %s" % regex)

				re_regex = re.compile(regex)
				for file in all_files:
					name =  file
					match = re_regex.match(name)
					if match:
						result.append(rel_path + name)
						matched.append(match.groups())
			else:
				result.append(rel_path + input)
		inputs = result

	if outputs:
		result = []
		for output in outputs:
			# we want \number instead of capture groups
			regex = wildcard_regex(output, True)
			if regex:
				for match in matched:
					# replace \number with data
					def replace_group(matchobj):
						index = int(matchobj.group(1)) - 1
						if index >= 0 and index < len(match):
							return match[index]
						else:
							return ""
					file = re_capture_group_ref.sub(replace_group, regex)
					file = re_non_escaped_char.sub(replace_non_esc, file)

					result.append(rel_path + file)
			else:
				result.append(rel_path + output)

		# normalize results
		result = [os.path.normpath(file).replace("\\", "/") for file in result]

	# normalize inputs
	inputs = [os.path.normpath(file).replace("\\", "/") for file in inputs]

	if outputs:
		return inputs, result
	else:
		return inputs

# finds the file in path
def which(cmd, mode = os.F_OK | os.X_OK, path = None):
	if sys.version_info[0:2] >= (3, 3):
		return shutil.which(cmd, mode, path)
	else:
		def _access_check(fn, mode):
			return (os.path.exists(fn) and os.access(fn, mode)
					and not os.path.isdir(fn))

		if os.path.dirname(cmd):
			if _access_check(cmd, mode):
				return cmd
			return None

		if path is None:
			path = os.environ.get("PATH", os.defpath)
		if not path:
			return None
		path = path.split(os.pathsep)

		if sys.platform == "win32":
			if not os.curdir in path:
				path.insert(0, os.curdir)
			pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
			if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
				files = [cmd]
			else:
				files = [cmd + ext for ext in pathext]
		else:
			files = [cmd]

		seen = set()
		for dir in path:
			normdir = os.path.normcase(dir)
			if not normdir in seen:
				seen.add(normdir)
				for thefile in files:
					name = os.path.join(dir, thefile)
					if _access_check(name, mode):
						return name
		return None
<|endoftext|>


<commit_before>from flask import Flask
# For Caching
from flask.ext.cache import Cache
# For SqlAlchemy
from flask.ext.sqlalchemy import SQLAlchemy
from flask import abort, send_from_directory, request
import config
import routes
import re

from params import PI_BASE_URL

app = Flask("sample-project", static_folder='media', static_url_path=PI_BASE_URL + 'media')
# For SqlAlchemy
app.config['SQLALCHEMY_DATABASE_URI'] = config.SQLALCHEMY_URL

# Cache configuration for locally served pages
# Check Configuring Flask-Cache section for more details
# TODO Put this in config
app.config['CACHE_TYPE'] = 'simple'
app.cache = Cache(app)

if config.SENTRY_DSN:
    from raven.contrib.flask import Sentry
    app.config['SENTRY_DSN'] = config.SENTRY_DSN
    sentry = Sentry(app)

db = SQLAlchemy(app)

def make_xsi1_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_xsi1_' +args).encode('utf-8')

def make_pi1_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    path = request.path
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_pi1_' + path + '_' +args).encode('utf-8')

def make_xsi3_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_xsi3_' +args).encode('utf-8')

@app.route('/radiodns/epg/XSI.xml')
@app.cache.cached(timeout=500, key_prefix='XSI1_')
def epg_1_xml():
    """Special call for EPG XSI v1.1 2013.10 RadioDNS"""
    # Specified by v1.1 2013.10 /radiodns/epg/XSI.xml
    # http://<host>:<port>/radiodns/epg/XSI.xml
    # http://radiodns.org/wp-content/uploads/2013/10/REPG01-v1.1.pdf

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:

        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        sp = None
        stations = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:
            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    stations = Station.query.filter_by(service_provider_id=sp.id, radioepg_enabled=True) #, fqdn_prefix=station)

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                stations = filter(lambda x: x.fqdn_prefix == station, stations)

        else:
            sp = ServiceProvider.query.filter_by(codops="EBU").order_by(ServiceProvider.codops).first()
            stations = Station.query.filter_by(radioepg_enabled=True)

        if not sp:
            abort(404)

        list = []

        if stations:
            for elem in stations:

                entries = []

                # Channels
                for elem2 in elem.channels.order_by(Channel.name).all():
                    if elem2.servicefollowingentry.active:
                        entries.append(elem2.servicefollowingentry.json)

                        if elem2.type_id == 'fm':  # For FM, also append with the country code
                            try:
                                data2 = elem2.servicefollowingentry.json

                                # Split the URI
                                uri_dp = data2['uri'].split(':', 2)
                                uri_p = uri_dp[1].split('.')

                                pi_code = uri_p[0]

                                # Get the ISO code form the picode
                                ecc = Ecc.query.filter_by(pi=pi_code[0].upper(), ecc=pi_code[1:].upper()).first()

                                uri_p[0] = ecc.iso.lower()

                                # Update the new URL
                                data2['uri'] = uri_dp[0] + ':' + '.'.join(uri_p)

                                # Add the service
                                entries.append(data2)

                            except:
                                pass

                # Custom entries
                for elem2 in elem.servicefollowingentries.order_by(GenericServiceFollowingEntry.channel_uri).all():
                    if elem2.active:
                        entries.append(elem2.json)

                if entries:
                    list.append([elem.json, entries])

        return Response(render_template('radioepg/servicefollowing/xml1.html', stations=list, service_provider=sp,
                                        creation_time=datetime.datetime.now().strftime(time_format)), mimetype='text/xml')

    # Else
    abort(404)
# Override Cache Key for XSI 1
epg_1_xml.make_cache_key = make_xsi1_hostname_cache_key

@app.route('/radiodns/spi/3.1/SI.xml')
@app.cache.cached(timeout=500, key_prefix='XSI3_')
def epg_3_xml():
    """Special call for EPG SI vV3.1.1 2015.01 ETSI xml"""
    # Specified by 3.1.1 /radiodns/spi/3.1/SI.xml
    # http://<host>:<port>/radiodns/spi/3.1/SI.xml
    # http://www.etsi.org/deliver/etsi_ts/102800_102899/102818/03.01.01_60/ts_102818v030101p.pdf

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:

        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        sp = None
        stations = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:
            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    stations = Station.query.filter_by(service_provider_id=sp.id, radioepg_enabled=True) #, fqdn_prefix=station)

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                stations = filter(lambda x: x.fqdn_prefix == station, stations)

        else:
            sp = ServiceProvider.query.filter_by(codops="EBU").order_by(ServiceProvider.codops).first()
            stations = Station.query.filter_by(radioepg_enabled=True)

        if not sp:
            abort(404)

        list = []

        if stations:
            for elem in stations:

                entries = []

                # Channels
                for elem2 in elem.channels.order_by(Channel.name).all():
                    if elem2.servicefollowingentry.active:
                        entries.append(elem2.servicefollowingentry.json)

                        if elem2.type_id == 'fm':  # For FM, also add with the country code
                            try:
                                data2 = elem2.servicefollowingentry.json

                                # Split the URI
                                uri_dp = data2['uri'].split(':', 2)
                                uri_p = uri_dp[1].split('.')

                                pi_code = uri_p[0]

                                # Get the ISO code form the picode
                                ecc = Ecc.query.filter_by(pi=pi_code[0].upper(), ecc=pi_code[1:].upper()).first()

                                uri_p[0] = ecc.iso.lower()

                                # Update the new URL
                                data2['uri'] = uri_dp[0] + ':' + '.'.join(uri_p)

                                # Add the service
                                entries.append(data2)

                            except:
                                pass

                # Custom entries
                for elem2 in elem.servicefollowingentries.order_by(GenericServiceFollowingEntry.channel_uri).all():
                    if elem2.active:
                        entries.append(elem2.json)

                if entries:
                    list.append([elem.json, entries])

        return Response(render_template('radioepg/servicefollowing/xml3.html', stations=list, service_provider=sp,
                                        creation_time=datetime.datetime.now().strftime(time_format)), mimetype='text/xml')

    # Else
    abort(404)
# Override Cache Key for XSI 3
epg_3_xml.make_cache_key = make_xsi3_hostname_cache_key

# Disabled temporarily due to performance issues
@app.route('/disabled_radiodns/logo/<int:id>/<int:w>/<int:h>/logo.png')
@app.cache.cached(timeout=500)
def logo(id, w, h):
    """Return a logo for a station"""

    from models import Station
    import os

    station = Station.query.filter(Station.id == id).first()

    if not station:
        abort(404)

    dest_file = 'media/uploads/radioepg/cache/S%s_W%s_H%s_L%s.png' % (str(int(id)), str(int(w)), str(int(h)), str(station.epg_picture_id) if station.epg_picture_id else 'B')

    if not os.path.isfile(dest_file):

        from PIL import Image
        size = (w, h)
        image = Image.open(station.epg_picture.filename if station.epg_picture else 'media/uploads/radioepg/default.png')
        image.thumbnail(size, Image.ANTIALIAS)
        background = Image.new('RGBA' if station.epg_picture else 'RGB', size, (255, 255, 255, 0))
        background.paste(image, ((size[0] - image.size[0]) / 2, (size[1] - image.size[1]) / 2))

        background.save(dest_file)

    return send_from_directory(".", dest_file)

@app.route('/radiodns/epg/<path:path>/<int:date>_PI.xml')
@app.cache.cached(timeout=500, key_prefix='PI1_')
def epg_sch_1_xml(path, date):
    """Special call for EPG scheduling xml"""

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:
        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        channels = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:

            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    channels = Channel.query.join(Station).filter(Station.service_provider_id == sp.id,
                                                                  Station.radioepg_enabled).all()

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                channels = filter(lambda x: x.station.fqdn_prefix == station, channels)

        else:
            channels = Channel.query.join(Station).filter(Station.radioepg_enabled).all()

        if not channels:
            abort(404)

        station = None
        topiced_path = '/topic/' + path
        station_channel = filter(lambda x: x.topic_no_slash == topiced_path, channels)
        if not station_channel:
            # Check for country code if FM
            csplitter = path.split('/')
            eccstr = ''
            if csplitter[0] == 'fm':
                # Find correct ECC
                ecc = Ecc.query.filter_by(iso=csplitter[1].upper()).first()
                if ecc:
                    eccstr = ('%s%s' % (ecc.pi, ecc.ecc)).lower()
                    csplitter[1] = eccstr
                    topiced_ecc_path = '/'.join(csplitter)
                    station_channel = filter(lambda x: x.topic_no_slash == topiced_ecc_path, channels)
                else:
                    eccstr = csplitter[1]

            if not station_channel:
                # Still no station so check with wildcard as well
                splitter = path.split('/')
                if eccstr:
                    splitter[1] = eccstr
                splitter[3] = '*'
                wild_ecc_path = '/topic/' + '/'.join(splitter)
                station_channel = filter(lambda x: x.topic_no_slash == wild_ecc_path, channels)

            if not station_channel:
                # Choose first with same PI
                psplitter = path.split('/')
                psplitter[3] = ''
                if eccstr:
                    psplitter[1] = eccstr
                best_path = '/topic/' + '/'.join(psplitter)
                station_channel = filter(lambda x: x.topic_no_slash.startswith(best_path), channels)

        if station_channel:
            # Found station
            station = station_channel[0].station

            import datetime
            time_format = '%Y-%m-%dT%H:%M:%S%z'

            today = datetime.date.today()
            start_date = datetime.datetime.combine(today - datetime.timedelta(days=today.weekday()), datetime.time())
            end_date = start_date + datetime.timedelta(days=6, hours=23, minutes=59, seconds=59)

            # Filter by date
            date_to_filter = datetime.datetime.strptime(str(date), "%Y%m%d").date()
            real_start_date = datetime.datetime.combine(date_to_filter, datetime.time())
            real_end_date = start_date + datetime.timedelta(days=6, hours=23, minutes=59, seconds=59)

            list = []

            for elem in station.schedules.all():
                elem.start_date = start_date

                if elem.date_of_start_time.date() == date_to_filter:
                    list.append(elem.json)

            return Response(render_template('radioepg/schedule/xml1.html', schedules=list,
                                            start_time=real_start_date.strftime(time_format),
                                            end_time=real_end_date.strftime(time_format),
                                            creation_time=datetime.datetime.now().strftime(time_format)),
                            mimetype='text/xml')

    abort(404)
    # return 'Not enabled'
# Override Cache Key for PI 1
epg_sch_1_xml.make_cache_key = make_pi1_hostname_cache_key

# Load remaining plug-it routes
def load_actions(act_mod):
    routes.load_routes(app, act_mod)
<commit_msg>Disable correct EPG<commit_after>from flask import Flask
# For Caching
from flask.ext.cache import Cache
# For SqlAlchemy
from flask.ext.sqlalchemy import SQLAlchemy
from flask import abort, send_from_directory, request
import config
import routes
import re

from params import PI_BASE_URL

app = Flask("sample-project", static_folder='media', static_url_path=PI_BASE_URL + 'media')
# For SqlAlchemy
app.config['SQLALCHEMY_DATABASE_URI'] = config.SQLALCHEMY_URL

# Cache configuration for locally served pages
# Check Configuring Flask-Cache section for more details
# TODO Put this in config
app.config['CACHE_TYPE'] = 'simple'
app.cache = Cache(app)

if config.SENTRY_DSN:
    from raven.contrib.flask import Sentry
    app.config['SENTRY_DSN'] = config.SENTRY_DSN
    sentry = Sentry(app)

db = SQLAlchemy(app)

def make_xsi1_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_xsi1_' +args).encode('utf-8')

def make_pi1_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    path = request.path
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_pi1_' + path + '_' +args).encode('utf-8')

def make_xsi3_hostname_cache_key(*args, **kwargs):
    """Generates a cachekey containing the hostname"""
    hostname = request.host
    args = str(hash(frozenset(request.args.items())))
    return (hostname + '_xsi3_' +args).encode('utf-8')

@app.route('/radiodns/epg/XSI.xml')
@app.cache.cached(timeout=500, key_prefix='XSI1_')
def epg_1_xml():
    """Special call for EPG XSI v1.1 2013.10 RadioDNS"""
    # Specified by v1.1 2013.10 /radiodns/epg/XSI.xml
    # http://<host>:<port>/radiodns/epg/XSI.xml
    # http://radiodns.org/wp-content/uploads/2013/10/REPG01-v1.1.pdf

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:

        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        sp = None
        stations = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:
            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    stations = Station.query.filter_by(service_provider_id=sp.id, radioepg_enabled=True) #, fqdn_prefix=station)

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                stations = filter(lambda x: x.fqdn_prefix == station, stations)

        else:
            sp = ServiceProvider.query.filter_by(codops="EBU").order_by(ServiceProvider.codops).first()
            stations = Station.query.filter_by(radioepg_enabled=True)

        if not sp:
            abort(404)

        list = []

        if stations:
            for elem in stations:

                entries = []

                # Channels
                for elem2 in elem.channels.order_by(Channel.name).all():
                    if elem2.servicefollowingentry.active:
                        entries.append(elem2.servicefollowingentry.json)

                        if elem2.type_id == 'fm':  # For FM, also append with the country code
                            try:
                                data2 = elem2.servicefollowingentry.json

                                # Split the URI
                                uri_dp = data2['uri'].split(':', 2)
                                uri_p = uri_dp[1].split('.')

                                pi_code = uri_p[0]

                                # Get the ISO code form the picode
                                ecc = Ecc.query.filter_by(pi=pi_code[0].upper(), ecc=pi_code[1:].upper()).first()

                                uri_p[0] = ecc.iso.lower()

                                # Update the new URL
                                data2['uri'] = uri_dp[0] + ':' + '.'.join(uri_p)

                                # Add the service
                                entries.append(data2)

                            except:
                                pass

                # Custom entries
                for elem2 in elem.servicefollowingentries.order_by(GenericServiceFollowingEntry.channel_uri).all():
                    if elem2.active:
                        entries.append(elem2.json)

                if entries:
                    list.append([elem.json, entries])

        return Response(render_template('radioepg/servicefollowing/xml1.html', stations=list, service_provider=sp,
                                        creation_time=datetime.datetime.now().strftime(time_format)), mimetype='text/xml')

    # Else
    abort(404)
# Override Cache Key for XSI 1
epg_1_xml.make_cache_key = make_xsi1_hostname_cache_key

@app.route('/radiodns/spi/3.1/SI.xml')
@app.cache.cached(timeout=500, key_prefix='XSI3_')
def epg_3_xml():
    """Special call for EPG SI vV3.1.1 2015.01 ETSI xml"""
    # Specified by 3.1.1 /radiodns/spi/3.1/SI.xml
    # http://<host>:<port>/radiodns/spi/3.1/SI.xml
    # http://www.etsi.org/deliver/etsi_ts/102800_102899/102818/03.01.01_60/ts_102818v030101p.pdf

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:

        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        sp = None
        stations = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:
            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    stations = Station.query.filter_by(service_provider_id=sp.id, radioepg_enabled=True) #, fqdn_prefix=station)

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                stations = filter(lambda x: x.fqdn_prefix == station, stations)

        else:
            sp = ServiceProvider.query.filter_by(codops="EBU").order_by(ServiceProvider.codops).first()
            stations = Station.query.filter_by(radioepg_enabled=True)

        if not sp:
            abort(404)

        list = []

        if stations:
            for elem in stations:

                entries = []

                # Channels
                for elem2 in elem.channels.order_by(Channel.name).all():
                    if elem2.servicefollowingentry.active:
                        entries.append(elem2.servicefollowingentry.json)

                        if elem2.type_id == 'fm':  # For FM, also add with the country code
                            try:
                                data2 = elem2.servicefollowingentry.json

                                # Split the URI
                                uri_dp = data2['uri'].split(':', 2)
                                uri_p = uri_dp[1].split('.')

                                pi_code = uri_p[0]

                                # Get the ISO code form the picode
                                ecc = Ecc.query.filter_by(pi=pi_code[0].upper(), ecc=pi_code[1:].upper()).first()

                                uri_p[0] = ecc.iso.lower()

                                # Update the new URL
                                data2['uri'] = uri_dp[0] + ':' + '.'.join(uri_p)

                                # Add the service
                                entries.append(data2)

                            except:
                                pass

                # Custom entries
                for elem2 in elem.servicefollowingentries.order_by(GenericServiceFollowingEntry.channel_uri).all():
                    if elem2.active:
                        entries.append(elem2.json)

                if entries:
                    list.append([elem.json, entries])

        return Response(render_template('radioepg/servicefollowing/xml3.html', stations=list, service_provider=sp,
                                        creation_time=datetime.datetime.now().strftime(time_format)), mimetype='text/xml')

    # Else
    abort(404)
# Override Cache Key for XSI 3
epg_3_xml.make_cache_key = make_xsi3_hostname_cache_key

# Disabled temporarily due to performance issues
@app.route('/radiodns/logo/<int:id>/<int:w>/<int:h>/logo.png')
@app.cache.cached(timeout=500)
def logo(id, w, h):
    """Return a logo for a station"""

    from models import Station
    import os

    station = Station.query.filter(Station.id == id).first()

    if not station:
        abort(404)

    dest_file = 'media/uploads/radioepg/cache/S%s_W%s_H%s_L%s.png' % (str(int(id)), str(int(w)), str(int(h)), str(station.epg_picture_id) if station.epg_picture_id else 'B')

    if not os.path.isfile(dest_file):

        from PIL import Image
        size = (w, h)
        image = Image.open(station.epg_picture.filename if station.epg_picture else 'media/uploads/radioepg/default.png')
        image.thumbnail(size, Image.ANTIALIAS)
        background = Image.new('RGBA' if station.epg_picture else 'RGB', size, (255, 255, 255, 0))
        background.paste(image, ((size[0] - image.size[0]) / 2, (size[1] - image.size[1]) / 2))

        background.save(dest_file)

    return send_from_directory(".", dest_file)

@app.route('/disabled_radiodns/epg/<path:path>/<int:date>_PI.xml')
@app.cache.cached(timeout=500, key_prefix='PI1_')
def epg_sch_1_xml(path, date):
    """Special call for EPG scheduling xml"""

    #        chrts.epg.radio.ebu.io >> XSI for all CHRTS Channels
    # la1ere.chrts.epg.radio.ebu.io >> XSI for La1ere only in CHRTS

    if config.XSISERVING_ENABLED:
        from models import Station, Channel, GenericServiceFollowingEntry, Ecc, ServiceProvider
        from flask import render_template, Response
        import datetime
        time_format = '%Y-%m-%dT%H:%M:%S%z'

        channels = None

        # Find out what stations to serve if by codops or station
        regex = re.compile('((?P<station>[^\.]+?)\.)?(?P<provider>[^\.]+?)\.'+config.XSISERVING_DOMAIN)
        r = regex.search(request.host)
        if r:

            station = r.groupdict()['station']
            provider = r.groupdict()['provider']

            if provider:
                # We have a station based query
                sp = ServiceProvider.query.filter_by(codops=provider).order_by(ServiceProvider.codops).first()
                if sp:
                    channels = Channel.query.join(Station).filter(Station.service_provider_id == sp.id,
                                                                  Station.radioepg_enabled).all()

            if station:
                # TODO FIX Filtering by property does not workin in SQLAlchemy, thus using regular python to filter
                channels = filter(lambda x: x.station.fqdn_prefix == station, channels)

        else:
            channels = Channel.query.join(Station).filter(Station.radioepg_enabled).all()

        if not channels:
            abort(404)

        station = None
        topiced_path = '/topic/' + path
        station_channel = filter(lambda x: x.topic_no_slash == topiced_path, channels)
        if not station_channel:
            # Check for country code if FM
            csplitter = path.split('/')
            eccstr = ''
            if csplitter[0] == 'fm':
                # Find correct ECC
                ecc = Ecc.query.filter_by(iso=csplitter[1].upper()).first()
                if ecc:
                    eccstr = ('%s%s' % (ecc.pi, ecc.ecc)).lower()
                    csplitter[1] = eccstr
                    topiced_ecc_path = '/'.join(csplitter)
                    station_channel = filter(lambda x: x.topic_no_slash == topiced_ecc_path, channels)
                else:
                    eccstr = csplitter[1]

            if not station_channel:
                # Still no station so check with wildcard as well
                splitter = path.split('/')
                if eccstr:
                    splitter[1] = eccstr
                splitter[3] = '*'
                wild_ecc_path = '/topic/' + '/'.join(splitter)
                station_channel = filter(lambda x: x.topic_no_slash == wild_ecc_path, channels)

            if not station_channel:
                # Choose first with same PI
                psplitter = path.split('/')
                psplitter[3] = ''
                if eccstr:
                    psplitter[1] = eccstr
                best_path = '/topic/' + '/'.join(psplitter)
                station_channel = filter(lambda x: x.topic_no_slash.startswith(best_path), channels)

        if station_channel:
            # Found station
            station = station_channel[0].station

            import datetime
            time_format = '%Y-%m-%dT%H:%M:%S%z'

            today = datetime.date.today()
            start_date = datetime.datetime.combine(today - datetime.timedelta(days=today.weekday()), datetime.time())
            end_date = start_date + datetime.timedelta(days=6, hours=23, minutes=59, seconds=59)

            # Filter by date
            date_to_filter = datetime.datetime.strptime(str(date), "%Y%m%d").date()
            real_start_date = datetime.datetime.combine(date_to_filter, datetime.time())
            real_end_date = start_date + datetime.timedelta(days=6, hours=23, minutes=59, seconds=59)

            list = []

            for elem in station.schedules.all():
                elem.start_date = start_date

                if elem.date_of_start_time.date() == date_to_filter:
                    list.append(elem.json)

            return Response(render_template('radioepg/schedule/xml1.html', schedules=list,
                                            start_time=real_start_date.strftime(time_format),
                                            end_time=real_end_date.strftime(time_format),
                                            creation_time=datetime.datetime.now().strftime(time_format)),
                            mimetype='text/xml')

    abort(404)
    # return 'Not enabled'
# Override Cache Key for PI 1
epg_sch_1_xml.make_cache_key = make_pi1_hostname_cache_key

# Load remaining plug-it routes
def load_actions(act_mod):
    routes.load_routes(app, act_mod)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from __future__ import annotations

import ast
import collections.abc
import itertools
import pathlib
import sys

PACKAGE_ROOT = pathlib.Path(__file__).resolve().parents[3].joinpath("airflow")
DAG_PY = PACKAGE_ROOT.joinpath("models", "dag.py")
UTILS_TG_PY = PACKAGE_ROOT.joinpath("utils", "task_group.py")
DECOS_TG_PY = PACKAGE_ROOT.joinpath("decorators", "task_group.py")

def _find_dag_init(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``DAG`` class's ``__init__``."""
    dag_class = next(n for n in ast.iter_child_nodes(mod) if isinstance(n, ast.ClassDef) and n.name == "DAG")
    return next(
        node
        for node in ast.iter_child_nodes(dag_class)
        if isinstance(node, ast.FunctionDef) and node.name == "__init__"
    )

def _find_dag_deco(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``@dag`` decorator."""
    return next(n for n in ast.iter_child_nodes(mod) if isinstance(n, ast.FunctionDef) and n.name == "dag")

def _find_tg_init(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``TaskGroup`` class's ``__init__``."""
    task_group_class = next(
        node
        for node in ast.iter_child_nodes(mod)
        if isinstance(node, ast.ClassDef) and node.name == "TaskGroup"
    )
    return next(
        node
        for node in ast.iter_child_nodes(task_group_class)
        if isinstance(node, ast.FunctionDef) and node.name == "__init__"
    )

def _find_tg_deco(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``@task_group`` decorator.

    The decorator has multiple overloads, but we want the first one, which
    contains task group init arguments.
    """
    return next(
        node
        for node in ast.iter_child_nodes(mod)
        if isinstance(node, ast.FunctionDef) and node.name == "task_group"
    )

# The new unparse() output is much more readable; fallback to dump() otherwise.
if sys.version_info >= (3, 8):
    _reveal = ast.unparse  # type: ignore[attr-defined]
else:
    _reveal = ast.dump

def _match_arguments(
    init_def: tuple[str, list[ast.arg]],
    deco_def: tuple[str, list[ast.arg]],
) -> collections.abc.Iterator[str]:
    init_name, init_args = init_def
    deco_name, deco_args = deco_def
    for i, (ini, dec) in enumerate(itertools.zip_longest(init_args, deco_args, fillvalue=None)):
        if ini is None and dec is not None:
            yield f"Argument present in @{deco_name} but missing from {init_name}: {dec.arg}"
            return
        if dec is None and ini is not None:
            yield f"Argument present in {init_name} but missing from @{deco_name}: {ini.arg}"
            return
        assert ini is not None and dec is not None  # Because None is only possible as fillvalue.

        if ini.arg != dec.arg:
            yield f"Argument {i + 1} mismatch: {init_name} has {ini.arg} but @{deco_name} has {dec.arg}"
            return

        if getattr(ini, "type_comment", None):  # 3.8+
            yield f"Do not use type comments on {init_name} argument: {ini.arg}"
        if getattr(dec, "type_comment", None):  # 3.8+
            yield f"Do not use type comments on @{deco_name} argument: {dec.arg}"

        # Poorly implemented node equality check.
        if ini.annotation and dec.annotation and ast.dump(ini.annotation) != ast.dump(dec.annotation):
            yield (
                f"Type annotations differ on argument {ini.arg} between {init_name} and @{deco_name}: "
                f"{_reveal(ini.annotation)} != {_reveal(dec.annotation)}"
            )
        else:
            if not ini.annotation:
                yield f"Type annotation missing on {init_name} argument: {ini.arg}"
            if not dec.annotation:
                yield f"Type annotation missing on @{deco_name} argument: {ini.arg}"

def _match_defaults(
    arg_names: list[str],
    init_def: tuple[str, list[ast.expr]],
    deco_def: tuple[str, list[ast.expr]],
) -> collections.abc.Iterator[str]:
    init_name, init_defaults = init_def
    deco_name, deco_defaults = deco_def
    for i, (ini, dec) in enumerate(zip(init_defaults, deco_defaults), 1):
        if ast.dump(ini) != ast.dump(dec):  # Poorly implemented equality check.
            yield (
                f"Argument {arg_names[i]!r} default mismatch: "
                f"{init_name} has {_reveal(ini)} but @{deco_name} has {_reveal(dec)}"
            )

def check_dag_init_decorator_arguments() -> int:
    dag_mod = ast.parse(DAG_PY.read_text("utf-8"), str(DAG_PY))

    utils_tg = ast.parse(UTILS_TG_PY.read_text("utf-8"), str(UTILS_TG_PY))
    decos_tg = ast.parse(DECOS_TG_PY.read_text("utf-8"), str(DECOS_TG_PY))

    items_to_check = [
        ("DAG", _find_dag_init(dag_mod), "dag", _find_dag_deco(dag_mod), "dag_id", ""),
        ("TaskGroup", _find_tg_init(utils_tg), "task_group", _find_tg_deco(decos_tg), "group_id", None),
    ]

    for init_name, init, deco_name, deco, id_arg, id_default in items_to_check:
        if getattr(init.args, "posonlyargs", None) or getattr(deco.args, "posonlyargs", None):
            print(f"{init_name} and @{deco_name} should not declare positional-only arguments")
            return -1
        if init.args.vararg or init.args.kwarg or deco.args.vararg or deco.args.kwarg:
            print(f"{init_name} and @{deco_name} should not declare *args and **kwargs")
            return -1

        # Feel free to change this and make some of the arguments keyword-only!
        if init.args.kwonlyargs or deco.args.kwonlyargs:
            print(f"{init_name}() and @{deco_name}() should not declare keyword-only arguments")
            return -2
        if init.args.kw_defaults or deco.args.kw_defaults:
            print(f"{init_name}() and @{deco_name}() should not declare keyword-only arguments")
            return -2

        init_arg_names = [a.arg for a in init.args.args]
        deco_arg_names = [a.arg for a in deco.args.args]

        if init_arg_names[0] != "self":
            print(f"First argument in {init_name} must be 'self'")
            return -3
        if init_arg_names[1] != id_arg:
            print(f"Second argument in {init_name} must be {id_arg!r}")
            return -3
        if deco_arg_names[0] != id_arg:
            print(f"First argument in @{deco_name} must be {id_arg!r}")
            return -3

        if len(init.args.defaults) != len(init_arg_names) - 2:
            print(f"All arguments on {init_name} except self and {id_arg} must have defaults")
            return -4
        if len(deco.args.defaults) != len(deco_arg_names):
            print(f"All arguments on @{deco_name} must have defaults")
            return -4
        if isinstance(deco.args.defaults[0], ast.Constant) and deco.args.defaults[0].value != id_default:
            print(f"Default {id_arg} on @{deco_name} must be {id_default!r}")
            return -4

    for init_name, init, deco_name, deco, _, _ in items_to_check:
        errors = list(_match_arguments((init_name, init.args.args[1:]), (deco_name, deco.args.args)))
        if errors:
            break
        init_defaults_def = (init_name, init.args.defaults)
        deco_defaults_def = (deco_name, deco.args.defaults[1:])
        errors = list(_match_defaults(deco_arg_names, init_defaults_def, deco_defaults_def))
        if errors:
            break

    for error in errors:
        print(error)
    return len(errors)

if __name__ == "__main__":
    sys.exit(check_dag_init_decorator_arguments())
<commit_msg>Add dynamic attr check for ast.unparse method in pre-commit (#27280)<commit_after>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from __future__ import annotations

import ast
import collections.abc
import itertools
import pathlib
import sys

PACKAGE_ROOT = pathlib.Path(__file__).resolve().parents[3].joinpath("airflow")
DAG_PY = PACKAGE_ROOT.joinpath("models", "dag.py")
UTILS_TG_PY = PACKAGE_ROOT.joinpath("utils", "task_group.py")
DECOS_TG_PY = PACKAGE_ROOT.joinpath("decorators", "task_group.py")

def _find_dag_init(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``DAG`` class's ``__init__``."""
    dag_class = next(n for n in ast.iter_child_nodes(mod) if isinstance(n, ast.ClassDef) and n.name == "DAG")
    return next(
        node
        for node in ast.iter_child_nodes(dag_class)
        if isinstance(node, ast.FunctionDef) and node.name == "__init__"
    )

def _find_dag_deco(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``@dag`` decorator."""
    return next(n for n in ast.iter_child_nodes(mod) if isinstance(n, ast.FunctionDef) and n.name == "dag")

def _find_tg_init(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``TaskGroup`` class's ``__init__``."""
    task_group_class = next(
        node
        for node in ast.iter_child_nodes(mod)
        if isinstance(node, ast.ClassDef) and node.name == "TaskGroup"
    )
    return next(
        node
        for node in ast.iter_child_nodes(task_group_class)
        if isinstance(node, ast.FunctionDef) and node.name == "__init__"
    )

def _find_tg_deco(mod: ast.Module) -> ast.FunctionDef:
    """Find definition of the ``@task_group`` decorator.

    The decorator has multiple overloads, but we want the first one, which
    contains task group init arguments.
    """
    return next(
        node
        for node in ast.iter_child_nodes(mod)
        if isinstance(node, ast.FunctionDef) and node.name == "task_group"
    )

# The new unparse() output is much more readable; fallback to dump() otherwise.
if hasattr(ast, "unparse"):
    _reveal = ast.unparse  # type: ignore[attr-defined]
else:
    _reveal = ast.dump

def _match_arguments(
    init_def: tuple[str, list[ast.arg]],
    deco_def: tuple[str, list[ast.arg]],
) -> collections.abc.Iterator[str]:
    init_name, init_args = init_def
    deco_name, deco_args = deco_def
    for i, (ini, dec) in enumerate(itertools.zip_longest(init_args, deco_args, fillvalue=None)):
        if ini is None and dec is not None:
            yield f"Argument present in @{deco_name} but missing from {init_name}: {dec.arg}"
            return
        if dec is None and ini is not None:
            yield f"Argument present in {init_name} but missing from @{deco_name}: {ini.arg}"
            return
        assert ini is not None and dec is not None  # Because None is only possible as fillvalue.

        if ini.arg != dec.arg:
            yield f"Argument {i + 1} mismatch: {init_name} has {ini.arg} but @{deco_name} has {dec.arg}"
            return

        if getattr(ini, "type_comment", None):  # 3.8+
            yield f"Do not use type comments on {init_name} argument: {ini.arg}"
        if getattr(dec, "type_comment", None):  # 3.8+
            yield f"Do not use type comments on @{deco_name} argument: {dec.arg}"

        # Poorly implemented node equality check.
        if ini.annotation and dec.annotation and ast.dump(ini.annotation) != ast.dump(dec.annotation):
            yield (
                f"Type annotations differ on argument {ini.arg} between {init_name} and @{deco_name}: "
                f"{_reveal(ini.annotation)} != {_reveal(dec.annotation)}"
            )
        else:
            if not ini.annotation:
                yield f"Type annotation missing on {init_name} argument: {ini.arg}"
            if not dec.annotation:
                yield f"Type annotation missing on @{deco_name} argument: {ini.arg}"

def _match_defaults(
    arg_names: list[str],
    init_def: tuple[str, list[ast.expr]],
    deco_def: tuple[str, list[ast.expr]],
) -> collections.abc.Iterator[str]:
    init_name, init_defaults = init_def
    deco_name, deco_defaults = deco_def
    for i, (ini, dec) in enumerate(zip(init_defaults, deco_defaults), 1):
        if ast.dump(ini) != ast.dump(dec):  # Poorly implemented equality check.
            yield (
                f"Argument {arg_names[i]!r} default mismatch: "
                f"{init_name} has {_reveal(ini)} but @{deco_name} has {_reveal(dec)}"
            )

def check_dag_init_decorator_arguments() -> int:
    dag_mod = ast.parse(DAG_PY.read_text("utf-8"), str(DAG_PY))

    utils_tg = ast.parse(UTILS_TG_PY.read_text("utf-8"), str(UTILS_TG_PY))
    decos_tg = ast.parse(DECOS_TG_PY.read_text("utf-8"), str(DECOS_TG_PY))

    items_to_check = [
        ("DAG", _find_dag_init(dag_mod), "dag", _find_dag_deco(dag_mod), "dag_id", ""),
        ("TaskGroup", _find_tg_init(utils_tg), "task_group", _find_tg_deco(decos_tg), "group_id", None),
    ]

    for init_name, init, deco_name, deco, id_arg, id_default in items_to_check:
        if getattr(init.args, "posonlyargs", None) or getattr(deco.args, "posonlyargs", None):
            print(f"{init_name} and @{deco_name} should not declare positional-only arguments")
            return -1
        if init.args.vararg or init.args.kwarg or deco.args.vararg or deco.args.kwarg:
            print(f"{init_name} and @{deco_name} should not declare *args and **kwargs")
            return -1

        # Feel free to change this and make some of the arguments keyword-only!
        if init.args.kwonlyargs or deco.args.kwonlyargs:
            print(f"{init_name}() and @{deco_name}() should not declare keyword-only arguments")
            return -2
        if init.args.kw_defaults or deco.args.kw_defaults:
            print(f"{init_name}() and @{deco_name}() should not declare keyword-only arguments")
            return -2

        init_arg_names = [a.arg for a in init.args.args]
        deco_arg_names = [a.arg for a in deco.args.args]

        if init_arg_names[0] != "self":
            print(f"First argument in {init_name} must be 'self'")
            return -3
        if init_arg_names[1] != id_arg:
            print(f"Second argument in {init_name} must be {id_arg!r}")
            return -3
        if deco_arg_names[0] != id_arg:
            print(f"First argument in @{deco_name} must be {id_arg!r}")
            return -3

        if len(init.args.defaults) != len(init_arg_names) - 2:
            print(f"All arguments on {init_name} except self and {id_arg} must have defaults")
            return -4
        if len(deco.args.defaults) != len(deco_arg_names):
            print(f"All arguments on @{deco_name} must have defaults")
            return -4
        if isinstance(deco.args.defaults[0], ast.Constant) and deco.args.defaults[0].value != id_default:
            print(f"Default {id_arg} on @{deco_name} must be {id_default!r}")
            return -4

    for init_name, init, deco_name, deco, _, _ in items_to_check:
        errors = list(_match_arguments((init_name, init.args.args[1:]), (deco_name, deco.args.args)))
        if errors:
            break
        init_defaults_def = (init_name, init.args.defaults)
        deco_defaults_def = (deco_name, deco.args.defaults[1:])
        errors = list(_match_defaults(deco_arg_names, init_defaults_def, deco_defaults_def))
        if errors:
            break

    for error in errors:
        print(error)
    return len(errors)

if __name__ == "__main__":
    sys.exit(check_dag_init_decorator_arguments())
<|endoftext|>


<commit_before>from os import remove
from discord.ext import commands
from bot import HahaNoUR
import math
from time import clock
from random import randint
from get_names import get_idol_names
from image_generator import create_image
from discord import User
from aliases import ALIASES

PAGE_SIZE = 12
IDOL_NAMES = get_idol_names()
SORTS = ["id", "name", "attribute", "rarity"]
RARITIES = ["UR", "SSR", "SR", "R", "N"]
ATRIBUTES = ["Smile", "Pure", "Cool"]

# Dictionary mapping user ids to last used album arguments
_last_user_args = {}

class AlbumCommands:
    """
    A class to hold all album commands.
    """

    def __init__(self, bot: HahaNoUR):
        self.bot = bot

    async def __send_error_msg(self, ctx):
        await self.bot.send_message(
            ctx.message.channel,
            '<@' + ctx.message.author.id + '> A transmission error occured.')

    async def __handle_result(self, ctx, album_size, image_path, delete=True):
        if not image_path:
            await self.__send_error_msg(ctx)
        else:
            page = _last_user_args[ctx.message.author.id]["page"]
            max_page = int(math.ceil(album_size / PAGE_SIZE))
            msg = '<@' + ctx.message.author.id + '> '
            msg += "Page " + str(page + 1) + " of " + str(max_page)
            await self.bot.upload(
                image_path, content=msg)

            if delete:
                remove(image_path)

    @commands.command(pass_context=True)
    @commands.cooldown(rate=3, per=2.5, type=commands.BucketType.user)
    async def album(self, ctx, *args: str):
        user = ctx.message.author
        album = self.bot.db.get_user_album(user)
        _parse_arguments(args, user)
        print(_last_user_args[user.id]["filter_type"])
        print(_last_user_args[user.id]["filter_value"])
        print(_last_user_args[user.id]["sort"])
        album = _apply_filter(album, user)
        album = _apply_sort(album, user)
        album_size = len(album)
        album = _splice_page(album, user)

        urls = []
        for card in album:
            if card["round_card_image"] is None:
                urls.append("http:" + card["round_card_idolized_image"])
            else:
                urls.append("http:" + card["round_card_image"])

        # TODO change this to call newer version of function that makes labels.
        if len(urls) > 0:
            image_path = await create_image(
                    urls, 2, str(clock()) + str(randint(0, 100)) + ".png")
        else:
            image_path = None

        await self.__handle_result(ctx, album_size, image_path)

def _apply_filter(album: list, user: User):
    """
    Applys a user's filters to a card album, removing anything not matching
        the filter.

    :param album: Album being filtered.
    :param user: User who requested the album.

    :return: Filtered album.
    """
    filter_type = _last_user_args[user.id]["filter_type"]
    filter_value = _last_user_args[user.id]["filter_value"]

    if filter_type == None:
        return album

    # Looping backwards since we are removing elements
    for i in range(len(album) - 1, -1, -1):
        if album[i][filter_type] != filter_value:
            album.pop(i)

    return album

def _apply_sort(album: list, user: User) -> list:
    """
    Applys a user's sort to a card album.

    :param album: Album being sorted.
    :param user: User who requested the album.

    :return: Sorted album.
    """
    sort = _last_user_args[user.id]["sort"]

    if sort == "rarity":
        album.sort(key=lambda x: RARITIES.index(x["rarity"]))
    elif sort == "attribute":
        album.sort(key=lambda x: ATRIBUTES.index(x["attribute"]))
    elif sort == "name":
        album.sort(key=lambda x: k["name"])

    return album

def _splice_page(album: list, user: User) -> list:
    """
    Splices a user's last requested page out of their album.

    :param album: Album being spliced
    :param user: User who requested the album.

    :return: Spliced album.
    """
    page = _last_user_args[user.id]["page"]
    max_page = int(math.ceil(len(album) / PAGE_SIZE)) - 1

    if page > max_page:
        page = max_page
    if page < 0:
        page = 0
    _last_user_args[user.id]["page"] = page

    start = PAGE_SIZE * page
    end = (PAGE_SIZE * page) + PAGE_SIZE
    return album[start:end]

def _parse_arguments(args: tuple, user: User):
    """
    Parse arguments to get how an album will be sorted and filtered. The parsed
        arguments are stored in the user's last used arguments dictionary.

    :param args: Tuple of arguments.
    :param user: User who requested the album.
    """
    # Add user to last used argument dictionary if they don't already exist.
    if not user.id in _last_user_args:
        _last_user_args[user.id] = {
            "page": 0,
            "filter_type": None, # Filter "all" if None
            "filter_value": None,
            "sort": None # Sort by ID if None
        }

    # Get values of user's last album preview.
    page = _last_user_args[user.id]["page"]
    filter_type = _last_user_args[user.id]["filter_type"]
    filter_value = _last_user_args[user.id]["filter_value"]
    sort = _last_user_args[user.id]["sort"]

    # Parse arguments
    for arg in args:
        arg = arg.lower()

        # Parse page number
        if _is_number(arg):
            # -1 since we start indexing at 0.
            page = int(arg) - 1

        # Parse filters
        if arg == "all":
            filter_type = None
            filter_value = None

        if arg.upper() in RARITIES:
            filter_type = "rarity"
            filter_value = arg.upper()

        if arg.title() in ATRIBUTES:
            filter_type = "attribute"
            filter_value = arg.title()

        # Check for names
        for full_name in IDOL_NAMES:
            name_split = full_name.split(" ")

            # Check if name is exact match
            if arg.title() == name_split[-1]:
                filter_type = "name"
                filter_value = full_name

        for key in ALIASES["name"]:
            if arg in key:
                filter_type = "name"
                filter_value = ALIASES["name"][key].title()

        # Parse sort
        if arg in SORTS:
            _last_user_args[user.id]["sort"] = arg

        _last_user_args[user.id]["page"] = page
        _last_user_args[user.id]["filter_type"] = filter_type
        _last_user_args[user.id]["filter_value"] = filter_value
        _last_user_args[user.id]["sort"] = sort

def _is_number(string: str) -> bool:
    """
    Checks if a string is a valid number.

    :param string: String being tested.

    :return: True if valid number, otherwise false.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False
<commit_msg>removed print statements<commit_after>from os import remove
from discord.ext import commands
from bot import HahaNoUR
import math
from time import clock
from random import randint
from get_names import get_idol_names
from image_generator import create_image
from discord import User
from aliases import ALIASES

PAGE_SIZE = 12
IDOL_NAMES = get_idol_names()
SORTS = ["id", "name", "attribute", "rarity"]
RARITIES = ["UR", "SSR", "SR", "R", "N"]
ATRIBUTES = ["Smile", "Pure", "Cool"]

# Dictionary mapping user ids to last used album arguments
_last_user_args = {}

class AlbumCommands:
    """
    A class to hold all album commands.
    """

    def __init__(self, bot: HahaNoUR):
        self.bot = bot

    async def __send_error_msg(self, ctx):
        await self.bot.send_message(
            ctx.message.channel,
            '<@' + ctx.message.author.id + '> A transmission error occured.')

    async def __handle_result(self, ctx, album_size, image_path, delete=True):
        if not image_path:
            await self.__send_error_msg(ctx)
        else:
            page = _last_user_args[ctx.message.author.id]["page"]
            max_page = int(math.ceil(album_size / PAGE_SIZE))
            msg = '<@' + ctx.message.author.id + '> '
            msg += "Page " + str(page + 1) + " of " + str(max_page)
            await self.bot.upload(
                image_path, content=msg)

            if delete:
                remove(image_path)

    @commands.command(pass_context=True)
    @commands.cooldown(rate=3, per=2.5, type=commands.BucketType.user)
    async def album(self, ctx, *args: str):
        user = ctx.message.author
        album = self.bot.db.get_user_album(user)
        _parse_arguments(args, user)
        album = _apply_filter(album, user)
        album = _apply_sort(album, user)
        album_size = len(album)
        album = _splice_page(album, user)

        urls = []
        for card in album:
            if card["round_card_image"] is None:
                urls.append("http:" + card["round_card_idolized_image"])
            else:
                urls.append("http:" + card["round_card_image"])

        # TODO change this to call newer version of function that makes labels.
        if len(urls) > 0:
            image_path = await create_image(
                    urls, 2, str(clock()) + str(randint(0, 100)) + ".png")
        else:
            image_path = None

        await self.__handle_result(ctx, album_size, image_path)

def _apply_filter(album: list, user: User):
    """
    Applys a user's filters to a card album, removing anything not matching
        the filter.

    :param album: Album being filtered.
    :param user: User who requested the album.

    :return: Filtered album.
    """
    filter_type = _last_user_args[user.id]["filter_type"]
    filter_value = _last_user_args[user.id]["filter_value"]

    if filter_type == None:
        return album

    # Looping backwards since we are removing elements
    for i in range(len(album) - 1, -1, -1):
        if album[i][filter_type] != filter_value:
            album.pop(i)

    return album

def _apply_sort(album: list, user: User) -> list:
    """
    Applys a user's sort to a card album.

    :param album: Album being sorted.
    :param user: User who requested the album.

    :return: Sorted album.
    """
    sort = _last_user_args[user.id]["sort"]

    if sort == "rarity":
        album.sort(key=lambda x: RARITIES.index(x["rarity"]))
    elif sort == "attribute":
        album.sort(key=lambda x: ATRIBUTES.index(x["attribute"]))
    elif sort == "name":
        album.sort(key=lambda x: k["name"])

    return album

def _splice_page(album: list, user: User) -> list:
    """
    Splices a user's last requested page out of their album.

    :param album: Album being spliced
    :param user: User who requested the album.

    :return: Spliced album.
    """
    page = _last_user_args[user.id]["page"]
    max_page = int(math.ceil(len(album) / PAGE_SIZE)) - 1

    if page > max_page:
        page = max_page
    if page < 0:
        page = 0
    _last_user_args[user.id]["page"] = page

    start = PAGE_SIZE * page
    end = (PAGE_SIZE * page) + PAGE_SIZE
    return album[start:end]

def _parse_arguments(args: tuple, user: User):
    """
    Parse arguments to get how an album will be sorted and filtered. The parsed
        arguments are stored in the user's last used arguments dictionary.

    :param args: Tuple of arguments.
    :param user: User who requested the album.
    """
    # Add user to last used argument dictionary if they don't already exist.
    if not user.id in _last_user_args:
        _last_user_args[user.id] = {
            "page": 0,
            "filter_type": None, # Filter "all" if None
            "filter_value": None,
            "sort": None # Sort by ID if None
        }

    # Get values of user's last album preview.
    page = _last_user_args[user.id]["page"]
    filter_type = _last_user_args[user.id]["filter_type"]
    filter_value = _last_user_args[user.id]["filter_value"]
    sort = _last_user_args[user.id]["sort"]

    # Parse arguments
    for arg in args:
        arg = arg.lower()

        # Parse page number
        if _is_number(arg):
            # -1 since we start indexing at 0.
            page = int(arg) - 1

        # Parse filters
        if arg == "all":
            filter_type = None
            filter_value = None

        if arg.upper() in RARITIES:
            filter_type = "rarity"
            filter_value = arg.upper()

        if arg.title() in ATRIBUTES:
            filter_type = "attribute"
            filter_value = arg.title()

        # Check for names
        for full_name in IDOL_NAMES:
            name_split = full_name.split(" ")

            # Check if name is exact match
            if arg.title() == name_split[-1]:
                filter_type = "name"
                filter_value = full_name

        for key in ALIASES["name"]:
            if arg in key:
                filter_type = "name"
                filter_value = ALIASES["name"][key].title()

        # Parse sort
        if arg in SORTS:
            _last_user_args[user.id]["sort"] = arg

        _last_user_args[user.id]["page"] = page
        _last_user_args[user.id]["filter_type"] = filter_type
        _last_user_args[user.id]["filter_value"] = filter_value
        _last_user_args[user.id]["sort"] = sort

def _is_number(string: str) -> bool:
    """
    Checks if a string is a valid number.

    :param string: String being tested.

    :return: True if valid number, otherwise false.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False
<|endoftext|>