<commit_before># Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import platform
import sys

# Make sure we're using Python3
assert sys.version.startswith('3'), "This script is only meant to work with Python3"

from .util import change_directory
from .util import clean_workspace
from .util import force_color
from .util import generated_venv_vars
from .util import info
from .util import log
from .util import UnbufferedIO

# Enforce unbuffered output
sys.stdout = UnbufferedIO(sys.stdout)
sys.stderr = UnbufferedIO(sys.stderr)

pip_dependencies = [
    'nose',
    'pep8',
    'pyflakes',
    'flake8',
    'mock',
    'coverage',
    'EmPy',
    'vcstool',
]

def main(sysargv=None):
    parser = argparse.ArgumentParser(
        description="Builds the ROS2 repositories as a single batch job")
    parser.add_argument(
        '--repo-file-url',
        default='https://raw.githubusercontent.com/ros2/examples/master/ros2.repos',
        help="url of the ros2.repos file to fetch and use for the basis of the batch job")
    parser.add_argument(
        '--test-branch', default=None,
        help="branch to attempt to checkout before doing batch job")
    parser.add_argument(
        '--white-space-in', nargs='*', default=None,
        choices=['sourcespace', 'buildspace', 'installspace', 'workspace'],
        help="which folder structures in which white space should be added")
    parser.add_argument(
        '--do-venv', default=False, action='store_true',
        help="create and use a virtual env in the build process")
    parser.add_argument(
        '--os', default=None, choices=['linux', 'osx', 'windows'])
    parser.add_argument(
        '--connext', default=False, action='store_true',
        help="try to build with connext")
    parser.add_argument(
        '--force-ansi-color', default=False, action='store_true',
        help="forces this program to output ansi color")

    args = parser.parse_args(sysargv)

    if args.force_ansi_color:
        force_color()

    job = None

    args.white_space_in = args.white_space_in or []
    args.workspace = 'work space' if 'workspace' in args.white_space_in else 'workspace'
    args.sourcespace = 'source space' if 'sourcespace' in args.white_space_in else 'src'
    args.buildspace = 'build space' if 'buildspace' in args.white_space_in else 'build'
    args.installspace = 'install space' if 'installspace' in args.white_space_in else 'install'

    platform_name = platform.platform().lower()
    if args.os == 'linux' or platform_name.startswith('linux'):
        args.os = 'linux'
        from .linux_batch import LinuxBatchJob
        job = LinuxBatchJob(args)
    elif args.os == 'osx' or platform_name.startswith('darwin'):
        args.os = 'osx'
        from .osx_batch import OSXBatchJob
        job = OSXBatchJob(args)
    elif args.os == 'windows' or platform_name.startswith('windows'):
        args.os = 'windows'
        from .windows_batch import WindowsBatchJob
        job = WindowsBatchJob(args)

    if args.do_venv and args.os == 'windows':
        sys.exit("--do-venv is not supported on windows")

    # Set the TERM env variable to coerce the output of Make to be colored.
    os.environ['TERM'] = os.environ.get('TERM', 'xterm-256color')

    info("Using workspace: @!{0}", fargs=(args.workspace,))
    clean_workspace(args.workspace)

    # Allow batch job to do OS specific stuff
    job.pre()
    # Check the env
    job.show_env()
    # Make sure virtual env is installed
    job.run([sys.executable, '-m', 'pip', 'install', '-U', 'virtualenv'])
    # Now inside of the workspace...
    with change_directory(args.workspace):
        # Enter a venv if asked to
        if args.do_venv:
            job.run([sys.executable, '-m', 'virtualenv', '-p', sys.executable, 'venv'])
            venv_path = os.path.abspath(os.path.join(os.getcwd(), 'venv'))
            venv, venv_python = generated_venv_vars(venv_path)
            job.push_run(venv)  # job.run is now venv
            job.push_python(venv_python)  # job.python is now venv_python
            job.show_env()
        # Update setuptools
        job.run([job.python, '-m', 'pip', 'install', '-U', 'pip', 'setuptools'])
        # Print setuptools version
        job.run([job.python, '-c', '"import setuptools; print(setuptools.__version__)"'])
        # Print the pip version
        job.run([job.python, '-m', 'pip', '--version'])
        # Install pip dependencies
        job.run([job.python, '-m', 'pip', 'install', '-U'] + pip_dependencies)
        # Get the repositories
        job.run(['curl', '-s', args.repo_file_url, '-o', 'ros2.repos'])
        # Show the contents
        log("@{bf}==>@| Contents of `ros2.repos`:")
        with open('ros2.repos', 'r') as f:
            print(f.read())
        # Use the repository listing and vcstool to fetch repositories
        if not os.path.exists(args.sourcespace):
            os.makedirs(args.sourcespace)
        job.run(['vcs', 'import', '"%s"' % args.sourcespace, '--input', 'ros2.repos'], shell=True)
        # Attempt to switch all the repositories to a given branch
        if args.test_branch is not None:
            info("Attempting to switch all repositories to the '{0}' branch"
                 .format(args.test_branch))
            vcs_custom_cmd = ['vcs', 'custom', '.', '--args', 'checkout', args.test_branch]
            ret = job.run(vcs_custom_cmd, exit_on_error=False)
            info("'{0}' returned exit code '{1}'", fargs=(vcs_custom_cmd, ret))
            print()
        # Allow the batch job to push custom sourcing onto the run command
        job.setup_env()
        ament_py = '"%s"' % os.path.join(
            '.', args.sourcespace, 'ament', 'ament_tools', 'scripts', 'ament.py'
        )
        # Now run ament build
        job.run([
            job.python, '-u', ament_py, 'build', '--build-tests',
            '--build-space', '"%s"' % args.buildspace,
            '--install-space', '"%s"' % args.installspace,
            '"%s"' % args.sourcespace
        ])
        # Run tests
        ret_test = job.run([
            job.python, '-u', ament_py, 'test',
            '--build-space', '"%s"' % args.buildspace,
            '--install-space', '"%s"' % args.installspace,
            '"%s"' % args.sourcespace
        ], exit_on_error=False)
        info("ament.py test returned: '{0}'".format(ret_test))
        # Collect the test results
        ret_test_results = job.run(
            [job.python, '-u', ament_py, 'test_results', '"%s"' % args.buildspace],
            exit_on_error=False
        )
        info("ament.py test_results returned: '{0}'".format(ret_test_results))
        # Uncomment this line to failing tests a failrue of this command.
        # return 0 if ret_test == 0 and ret_testr == 0 else 1
        return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>fix printing of setuptools version<commit_after># Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import platform
import sys

# Make sure we're using Python3
assert sys.version.startswith('3'), "This script is only meant to work with Python3"

from .util import change_directory
from .util import clean_workspace
from .util import force_color
from .util import generated_venv_vars
from .util import info
from .util import log
from .util import UnbufferedIO

# Enforce unbuffered output
sys.stdout = UnbufferedIO(sys.stdout)
sys.stderr = UnbufferedIO(sys.stderr)

pip_dependencies = [
    'nose',
    'pep8',
    'pyflakes',
    'flake8',
    'mock',
    'coverage',
    'EmPy',
    'vcstool',
]

def main(sysargv=None):
    parser = argparse.ArgumentParser(
        description="Builds the ROS2 repositories as a single batch job")
    parser.add_argument(
        '--repo-file-url',
        default='https://raw.githubusercontent.com/ros2/examples/master/ros2.repos',
        help="url of the ros2.repos file to fetch and use for the basis of the batch job")
    parser.add_argument(
        '--test-branch', default=None,
        help="branch to attempt to checkout before doing batch job")
    parser.add_argument(
        '--white-space-in', nargs='*', default=None,
        choices=['sourcespace', 'buildspace', 'installspace', 'workspace'],
        help="which folder structures in which white space should be added")
    parser.add_argument(
        '--do-venv', default=False, action='store_true',
        help="create and use a virtual env in the build process")
    parser.add_argument(
        '--os', default=None, choices=['linux', 'osx', 'windows'])
    parser.add_argument(
        '--connext', default=False, action='store_true',
        help="try to build with connext")
    parser.add_argument(
        '--force-ansi-color', default=False, action='store_true',
        help="forces this program to output ansi color")

    args = parser.parse_args(sysargv)

    if args.force_ansi_color:
        force_color()

    job = None

    args.white_space_in = args.white_space_in or []
    args.workspace = 'work space' if 'workspace' in args.white_space_in else 'workspace'
    args.sourcespace = 'source space' if 'sourcespace' in args.white_space_in else 'src'
    args.buildspace = 'build space' if 'buildspace' in args.white_space_in else 'build'
    args.installspace = 'install space' if 'installspace' in args.white_space_in else 'install'

    platform_name = platform.platform().lower()
    if args.os == 'linux' or platform_name.startswith('linux'):
        args.os = 'linux'
        from .linux_batch import LinuxBatchJob
        job = LinuxBatchJob(args)
    elif args.os == 'osx' or platform_name.startswith('darwin'):
        args.os = 'osx'
        from .osx_batch import OSXBatchJob
        job = OSXBatchJob(args)
    elif args.os == 'windows' or platform_name.startswith('windows'):
        args.os = 'windows'
        from .windows_batch import WindowsBatchJob
        job = WindowsBatchJob(args)

    if args.do_venv and args.os == 'windows':
        sys.exit("--do-venv is not supported on windows")

    # Set the TERM env variable to coerce the output of Make to be colored.
    os.environ['TERM'] = os.environ.get('TERM', 'xterm-256color')

    info("Using workspace: @!{0}", fargs=(args.workspace,))
    clean_workspace(args.workspace)

    # Allow batch job to do OS specific stuff
    job.pre()
    # Check the env
    job.show_env()
    # Make sure virtual env is installed
    job.run([sys.executable, '-m', 'pip', 'install', '-U', 'virtualenv'])
    # Now inside of the workspace...
    with change_directory(args.workspace):
        # Enter a venv if asked to
        if args.do_venv:
            job.run([sys.executable, '-m', 'virtualenv', '-p', sys.executable, 'venv'])
            venv_path = os.path.abspath(os.path.join(os.getcwd(), 'venv'))
            venv, venv_python = generated_venv_vars(venv_path)
            job.push_run(venv)  # job.run is now venv
            job.push_python(venv_python)  # job.python is now venv_python
            job.show_env()
        # Update setuptools
        job.run([job.python, '-m', 'pip', 'install', '-U', 'pip', 'setuptools'])
        # Print setuptools version
        job.run([job.python, '-c', '"import setuptools; print(setuptools.__version__)"'],
                shell=True)
        # Print the pip version
        job.run([job.python, '-m', 'pip', '--version'])
        # Install pip dependencies
        job.run([job.python, '-m', 'pip', 'install', '-U'] + pip_dependencies)
        # Get the repositories
        job.run(['curl', '-s', args.repo_file_url, '-o', 'ros2.repos'])
        # Show the contents
        log("@{bf}==>@| Contents of `ros2.repos`:")
        with open('ros2.repos', 'r') as f:
            print(f.read())
        # Use the repository listing and vcstool to fetch repositories
        if not os.path.exists(args.sourcespace):
            os.makedirs(args.sourcespace)
        job.run(['vcs', 'import', '"%s"' % args.sourcespace, '--input', 'ros2.repos'], shell=True)
        # Attempt to switch all the repositories to a given branch
        if args.test_branch is not None:
            info("Attempting to switch all repositories to the '{0}' branch"
                 .format(args.test_branch))
            vcs_custom_cmd = ['vcs', 'custom', '.', '--args', 'checkout', args.test_branch]
            ret = job.run(vcs_custom_cmd, exit_on_error=False)
            info("'{0}' returned exit code '{1}'", fargs=(vcs_custom_cmd, ret))
            print()
        # Allow the batch job to push custom sourcing onto the run command
        job.setup_env()
        ament_py = '"%s"' % os.path.join(
            '.', args.sourcespace, 'ament', 'ament_tools', 'scripts', 'ament.py'
        )
        # Now run ament build
        job.run([
            job.python, '-u', ament_py, 'build', '--build-tests',
            '--build-space', '"%s"' % args.buildspace,
            '--install-space', '"%s"' % args.installspace,
            '"%s"' % args.sourcespace
        ])
        # Run tests
        ret_test = job.run([
            job.python, '-u', ament_py, 'test',
            '--build-space', '"%s"' % args.buildspace,
            '--install-space', '"%s"' % args.installspace,
            '"%s"' % args.sourcespace
        ], exit_on_error=False)
        info("ament.py test returned: '{0}'".format(ret_test))
        # Collect the test results
        ret_test_results = job.run(
            [job.python, '-u', ament_py, 'test_results', '"%s"' % args.buildspace],
            exit_on_error=False
        )
        info("ament.py test_results returned: '{0}'".format(ret_test_results))
        # Uncomment this line to failing tests a failrue of this command.
        # return 0 if ret_test == 0 and ret_testr == 0 else 1
        return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>__author__ = 'cephalopodblue'
import os
import MBInfo
import argparse
import mutagen
import shutil
import JsonSerializer
import DaletSerializer
import csv

_file_types = {".wma": "wma", ".m4a": "aac", ".mp3": "id3", ".flac": "vorbis"}

def process_directory(source_dir, output_dir, serializer, delete_processed):
    cached_mb_releases = {}
    unique_artists = {}
    current_release_id = ''

    track_meta_dir = os.path.join(output_dir, 'track_meta')
    if not os.path.exists(track_meta_dir):
        os.makedirs(track_meta_dir)
    artist_meta_dir = os.path.join(output_dir, 'artist_meta')
    if not os.path.exists(artist_meta_dir):
        os.makedirs(artist_meta_dir)
    release_meta_dir = os.path.join(output_dir, 'release_meta')
    if not os.path.exists(release_meta_dir):
        os.makedirs(release_meta_dir)
    track_dir = os.path.join(output_dir, 'track')
    if not os.path.exists(track_dir):
        os.makedirs(track_dir)

    track_success_dir = os.path.join(output_dir, 'found')
    if not os.path.exists(track_success_dir):
        os.makedirs(track_success_dir)
    print("Track Success: ", track_success_dir)

    track_fail_dir = os.path.join(output_dir, 'not_found')
    if not os.path.exists(track_fail_dir):
        os.makedirs(track_fail_dir)
    print("Track Fail: ", track_fail_dir)

    path_start = len(source_dir) + 1
    for root, dir, files in os.walk(source_dir):
        if len(root) > path_start:
            path = root[path_start:]
        else:
            path = ''
        print(path)
        for src_name in files:
            file_name = os.path.join(root, src_name)
            copy_to_path = ''
            ext = os.path.splitext(file_name)[1].lower()
            if ext in _file_types:

                # Get the MusicBrainz Release ID from the file
                raw_metadata = mutagen.File(file_name)
                release_id = ''
                kexp_obscenity_rating = ''
                kexp_category = ''
                if _file_types[ext] == "vorbis":
                    if "mbid" in raw_metadata:
                        release_id = raw_metadata["mbid"][0]
                        track_num = int(raw_metadata["tracknumber"][0]) - 1
                        disc_num = int(raw_metadata["discnumber"][0])
                    elif "musicbrainz_albumid" in raw_metadata:
                        release_id = raw_metadata["musicbrainz_albumid"][0]
                        track_num = int(raw_metadata["tracknumber"][0]) - 1
                        disc_num =  int(raw_metadata["discnumber"][0])

                elif _file_types[ext] == "aac":
                    #raw_metadata.tags._DictProxy__dict['----:com.apple.iTunes:MBID']
                    raw_metadata = raw_metadata.tags._DictProxy__dict
                    if '----:com.apple.iTunes:MBID' in raw_metadata:
                        release_id = str(raw_metadata['----:com.apple.iTunes:MBID'][0])
                        track_num = int(raw_metadata['trkn'][0][0]) - 1
                        disc_num = int(raw_metadata['disk'][0][0])
                    elif '----:com.apple.iTunes:MusicBrainz Album Id' in raw_metadata:
                        release_id = str(raw_metadata['----:com.apple.iTunes:MusicBrainz Album Id'][0])
                        track_num = int(raw_metadata['trkn'][0][0]) - 1
                        disc_num = int(raw_metadata['disk'][0][0])
                    if '----:com.apple.iTunes:KEXPPRIMARYGENRE' in raw_metadata:
                        kexp_category = raw_metadata['----:com.apple.iTunes:KEXPPRIMARYGENRE'][0]
                    if '----:com.apple.iTunes:KEXPOBSCENITYRATING' in raw_metadata:
                        kexp_category = raw_metadata['----:com.apple.iTunes:KEXPOBSCENITYRATING'][0]

                elif _file_types[ext] == "id3":
                    raw_metadata = raw_metadata.tags._DictProxy__dict
                    if 'TXXX:MBID' in raw_metadata:
                        release_id = raw_metadata['TXXX:MBID'].text[0]
                        track_num = int(raw_metadata['TRCK'].text[0].split('/')[0]) - 1
                        disc_num = int(raw_metadata['TPOS'].text[0].split('/')[0])
                    elif 'TXXX:MusicBrainz Album Id' in raw_metadata:
                        release_id = raw_metadata['TXXX:MusicBrainz Album Id'].text[0]
                        track_num = int(raw_metadata['TRCK'].text[0].split('/')[0]) - 1
                        disc_num = int(raw_metadata['TPOS'].text[0].split('/')[0])

                if release_id > '':
                    print("Processing " + ascii(file_name))
                    try:
                        # Check if this is a new release (generally means we are in a new directory)
                        if release_id != current_release_id:
                            current_release_id = release_id

                            #See if we have already cached the release, or we need to pull if from MusicBrainz
                            if release_id in cached_mb_releases:
                                mb_release = cached_mb_releases[release_id]
                            else:
                                mb_release = MBInfo.get_release(release_id)
                                cached_mb_releases[release_id] = mb_release
                                release = MBInfo.process_release(mb_release, disc_num)
                                serializer.save_release(release, release_meta_dir)

                        # Pull metadata from MusicBrainz
                        track_data = MBInfo.process_track(mb_release, disc_num, track_num)

                        # Add KEXP added metadata from tags
                        track_data['kexp_category'] = kexp_category
                        track_data['kexp_obscenity_rating'] = kexp_obscenity_rating

                        # Extract artist information

                        # Save the metadata
                        serializer.save_track(raw_metadata, release, track_data, file_name, track_meta_dir)

                        # Copy files to to success directory
                        target = os.path.join(track_dir, track_data["track_id"] + ext)
                        shutil.copy(file_name, target)

                        # Make a backup of original file just in case
                        copy_to_path = os.path.join(track_success_dir, path)
                        shutil.copy(file_name, target)

                        # Add any new artist to our unique artists list
                        for artist in track_data["artist-credit"]:
                            if 'artist' in artist:
                                a = artist['artist']
                                artist_id = a['id']
                                if not (artist_id in unique_artists):
                                    unique_artists[artist_id] = a['name']
                                    serializer.save_artist(MBInfo.get_artist(artist_id), artist_meta_dir)

                    except UnicodeDecodeError:
                        print("    ERROR: Invalid characters!")
                else:
                    print("Skipping " + ascii(file_name))
                    copy_to_path = os.path.join(track_fail_dir, path)

                # Move the file out of the source directory
                if copy_to_path > '':
                    if not os.path.exists(copy_to_path):
                        os.makedirs(copy_to_path)
                    target = os.path.join(copy_to_path, src_name)
                    if delete_processed:
                        shutil.move(file_name, target)
                    else:
                        shutil.copy(file_name, target)

    # Write out our list of unique artists
    artist_fn = os.path.join(artist_meta_dir, 'unique_artists.csv')
    # with open(artist_fn, 'wb') as csvfile:
    #    artist_writer = csv.writer(csvfile)
    #    for artist_id, artist_name in unique_artists.iteritems():
    #        print(artist_id + ', ' + artist_name)
    #        artist_writer.writerow([artist_id.encode('utf-8'), artist_name.encode('utf-8', 'ignore')])

def main():
    """
    Crawls the given directory for audio files (currently only processes FLAC) and
    extracts raw metadata (expected to be acquired by ripping using dBpoweramp) and
    metadata from musicbrainz, json-formatted.

    Currently will only correctly process flac files (or other
    """
    parser = argparse.ArgumentParser(description='Get metadata from files.')
    parser.add_argument('input_directory', help="Input audio file.")
    parser.add_argument('output_directory', help="Directory to store output files. MUST ALREADY EXIST for now.")
    args = parser.parse_args()
    process_directory(args.input_directory, args.output_directory, DaletSerializer, False)

main()
<commit_msg>Fixed a byte to string error<commit_after>__author__ = 'cephalopodblue'
import os
import MBInfo
import argparse
import mutagen
import shutil
import JsonSerializer
import DaletSerializer
import csv

_file_types = {".wma": "wma", ".m4a": "aac", ".mp3": "id3", ".flac": "vorbis"}

def process_directory(source_dir, output_dir, serializer, delete_processed):
    cached_mb_releases = {}
    unique_artists = {}
    current_release_id = ''

    track_meta_dir = os.path.join(output_dir, 'track_meta')
    if not os.path.exists(track_meta_dir):
        os.makedirs(track_meta_dir)
    artist_meta_dir = os.path.join(output_dir, 'artist_meta')
    if not os.path.exists(artist_meta_dir):
        os.makedirs(artist_meta_dir)
    release_meta_dir = os.path.join(output_dir, 'release_meta')
    if not os.path.exists(release_meta_dir):
        os.makedirs(release_meta_dir)
    track_dir = os.path.join(output_dir, 'track')
    if not os.path.exists(track_dir):
        os.makedirs(track_dir)

    track_success_dir = os.path.join(output_dir, 'found')
    if not os.path.exists(track_success_dir):
        os.makedirs(track_success_dir)
    print("Track Success: ", track_success_dir)

    track_fail_dir = os.path.join(output_dir, 'not_found')
    if not os.path.exists(track_fail_dir):
        os.makedirs(track_fail_dir)
    print("Track Fail: ", track_fail_dir)

    path_start = len(source_dir) + 1
    for root, dir, files in os.walk(source_dir):
        if len(root) > path_start:
            path = root[path_start:]
        else:
            path = ''
        print(path)
        for src_name in files:
            file_name = os.path.join(root, src_name)
            copy_to_path = ''
            ext = os.path.splitext(file_name)[1].lower()
            if ext in _file_types:

                # Get the MusicBrainz Release ID from the file
                raw_metadata = mutagen.File(file_name)
                release_id = ''
                kexp_obscenity_rating = ''
                kexp_category = ''
                if _file_types[ext] == "vorbis":
                    if "mbid" in raw_metadata:
                        release_id = raw_metadata["mbid"][0]
                        track_num = int(raw_metadata["tracknumber"][0]) - 1
                        disc_num = int(raw_metadata["discnumber"][0])
                    elif "musicbrainz_albumid" in raw_metadata:
                        release_id = raw_metadata["musicbrainz_albumid"][0]
                        track_num = int(raw_metadata["tracknumber"][0]) - 1
                        disc_num =  int(raw_metadata["discnumber"][0])

                elif _file_types[ext] == "aac":
                    #raw_metadata.tags._DictProxy__dict['----:com.apple.iTunes:MBID']
                    raw_metadata = raw_metadata.tags._DictProxy__dict
                    if '----:com.apple.iTunes:MBID' in raw_metadata:
                        release_id = str(raw_metadata['----:com.apple.iTunes:MBID'][0])
                        track_num = int(raw_metadata['trkn'][0][0]) - 1
                        disc_num = int(raw_metadata['disk'][0][0])
                    elif '----:com.apple.iTunes:MusicBrainz Album Id' in raw_metadata:
                        release_id = str(raw_metadata['----:com.apple.iTunes:MusicBrainz Album Id'][0], encoding='UTF-8')
                        track_num = int(raw_metadata['trkn'][0][0]) - 1
                        disc_num = int(raw_metadata['disk'][0][0])
                    if '----:com.apple.iTunes:KEXPPRIMARYGENRE' in raw_metadata:
                        kexp_category = raw_metadata['----:com.apple.iTunes:KEXPPRIMARYGENRE'][0]
                    if '----:com.apple.iTunes:KEXPOBSCENITYRATING' in raw_metadata:
                        kexp_category = raw_metadata['----:com.apple.iTunes:KEXPOBSCENITYRATING'][0]

                elif _file_types[ext] == "id3":
                    raw_metadata = raw_metadata.tags._DictProxy__dict
                    if 'TXXX:MBID' in raw_metadata:
                        release_id = raw_metadata['TXXX:MBID'].text[0]
                        track_num = int(raw_metadata['TRCK'].text[0].split('/')[0]) - 1
                        disc_num = int(raw_metadata['TPOS'].text[0].split('/')[0])
                    elif 'TXXX:MusicBrainz Album Id' in raw_metadata:
                        release_id = raw_metadata['TXXX:MusicBrainz Album Id'].text[0]
                        track_num = int(raw_metadata['TRCK'].text[0].split('/')[0]) - 1
                        disc_num = int(raw_metadata['TPOS'].text[0].split('/')[0])

                if release_id > '':
                    print("Processing " + ascii(file_name))
                    try:
                        # Check if this is a new release (generally means we are in a new directory)
                        if release_id != current_release_id:
                            current_release_id = release_id

                            #See if we have already cached the release, or we need to pull if from MusicBrainz
                            if release_id in cached_mb_releases:
                                mb_release = cached_mb_releases[release_id]
                            else:
                                mb_release = MBInfo.get_release(release_id)
                                cached_mb_releases[release_id] = mb_release
                                release = MBInfo.process_release(mb_release, disc_num)
                                serializer.save_release(release, release_meta_dir)

                        # Pull metadata from MusicBrainz
                        track_data = MBInfo.process_track(mb_release, disc_num, track_num)

                        # Add KEXP added metadata from tags
                        track_data['kexp_category'] = kexp_category
                        track_data['kexp_obscenity_rating'] = kexp_obscenity_rating

                        # Extract artist information

                        # Save the metadata
                        serializer.save_track(raw_metadata, release, track_data, file_name, track_meta_dir)

                        # Copy files to to success directory
                        target = os.path.join(track_dir, track_data["track_id"] + ext)
                        shutil.copy(file_name, target)

                        # Make a backup of original file just in case
                        copy_to_path = os.path.join(track_success_dir, path)
                        shutil.copy(file_name, target)

                        # Add any new artist to our unique artists list
                        for artist in track_data["artist-credit"]:
                            if 'artist' in artist:
                                a = artist['artist']
                                artist_id = a['id']
                                if not (artist_id in unique_artists):
                                    unique_artists[artist_id] = a['name']
                                    serializer.save_artist(MBInfo.get_artist(artist_id), artist_meta_dir)

                    except UnicodeDecodeError:
                        print("    ERROR: Invalid characters!")
                else:
                    print("Skipping " + ascii(file_name))
                    copy_to_path = os.path.join(track_fail_dir, path)

                # Move the file out of the source directory
                if copy_to_path > '':
                    if not os.path.exists(copy_to_path):
                        os.makedirs(copy_to_path)
                    target = os.path.join(copy_to_path, src_name)
                    if delete_processed:
                        shutil.move(file_name, target)
                    else:
                        shutil.copy(file_name, target)

    # Write out our list of unique artists
    artist_fn = os.path.join(artist_meta_dir, 'unique_artists.csv')
    # with open(artist_fn, 'wb') as csvfile:
    #    artist_writer = csv.writer(csvfile)
    #    for artist_id, artist_name in unique_artists.iteritems():
    #        print(artist_id + ', ' + artist_name)
    #        artist_writer.writerow([artist_id.encode('utf-8'), artist_name.encode('utf-8', 'ignore')])

def main():
    """
    Crawls the given directory for audio files (currently only processes FLAC) and
    extracts raw metadata (expected to be acquired by ripping using dBpoweramp) and
    metadata from musicbrainz, json-formatted.

    Currently will only correctly process flac files (or other
    """
    parser = argparse.ArgumentParser(description='Get metadata from files.')
    parser.add_argument('input_directory', help="Input audio file.")
    parser.add_argument('output_directory', help="Directory to store output files. MUST ALREADY EXIST for now.")
    args = parser.parse_args()
    process_directory(args.input_directory, args.output_directory, DaletSerializer, False)

main()
<|endoftext|>


<commit_before>from collections import defaultdict
from os.path import expanduser
import traceback
import time
import os
import csv
import testing

class TestLogger:
    def __init__(self):
        log_dir, symlink_name, log_file_name = self._log_name()
        print('Creating new log:', log_file_name)
        os.makedirs(log_dir, exist_ok=True)
        try:
            os.remove(symlink_name)
        except: pass
        os.symlink(log_file_name, symlink_name)
        self.log_file = open(symlink_name, 'w')
        self.log = csv.writer(self.log_file, delimiter=',', quotechar='"')

    def _log_name(self):
        log_dir = expanduser('~/rstem_logs/')
        symlink_name = log_dir + 'testlog.txt'
        log_file_name = 'testlog.' + str(int(time.time())) + '.txt'
        return (log_dir, symlink_name, log_file_name)

    def write(self, test, test_type, exc):
        filename = line_num = func_name = line_text = exc_name = exc_msg = ''
        if exc:
            extracted_tb = traceback.extract_tb(exc.__traceback__)
            try:
                filename, line_num, func_name, line_text = list(extracted_tb[-1])
            except IndexError:
                pass
            exc_name = type(exc).__name__
            exc_msg = str(exc)

        test_func_name = str(test.__name__) if test else ""

        self.log.writerow([
            test_func_name,
            test_type,
            exc_name,
            filename,
            line_num,
            func_name,
            exc_msg
            ])

    def keyboard_interrupt(self):
        self.write(
            None,
            None,
            KeyboardInterrupt())

    def close(self):
        self.log_file.close()

    def print_summary(self):
        log_dir, symlink_name, log_file_name = self._log_name()

        # For each line in log

        with open(symlink_name) as log:
            class PassCounts:
                passed = 0
                failed = 0
                skipped = 0
                total = 0
            pass_counts_per_test_type = {}
            for t in testing.ordered_test_types:
                pass_counts_per_test_type[t] = PassCounts()
            for row in csv.reader(log):
                test_name, test_type, exc_name, filename, line_num, func_name, exc_msg = row
                if test_type:
                    if not exc_name:
                        pass_counts_per_test_type[test_type].passed += 1
                    elif exc_name == testing.TestSkippedException.__name__:
                        pass_counts_per_test_type[test_type].skipped += 1
                    else:
                        pass_counts_per_test_type[test_type].failed += 1
                    pass_counts_per_test_type[test_type].total += 1
            print("TEST SUMMARY:")
            for t in testing.ordered_test_types:
                if pass_counts_per_test_type[t].total > 0:
                    if pass_counts_per_test_type[t].passed == pass_counts_per_test_type[t].total:
                        print('\t{}: PASSED ALL TESTS ({})'.format(
                            t,
                            pass_counts_per_test_type[t].total,
                            ))
                    elif pass_counts_per_test_type[t].skipped == 0:
                        print('\t{}: passed {} of {}'.format(
                            t,
                            pass_counts_per_test_type[t].passed,
                            pass_counts_per_test_type[t].total,
                            ))
                    else:
                        print('\t{}: passed {} of {} (skipped {})'.format(
                            t,
                            pass_counts_per_test_type[t].passed,
                            pass_counts_per_test_type[t].total,
                            pass_counts_per_test_type[t].skipped,
                            ))
                expected_total = 0
                expected_total += pass_counts_per_test_type[t].passed
                expected_total += pass_counts_per_test_type[t].failed
                expected_total += pass_counts_per_test_type[t].skipped
                if expected_total != pass_counts_per_test_type[t].total:
                    print('WARNING: Counts don\'t add to total')

    def print_result_table(self):
        log_dir, symlink_name, log_file_name = self._log_name()
        base_fmt = '{pass_fail_char:2}{test_type:5}{test_name:30}'

        # Header - 2 lines: line of labels, then line of "-" for each label.
        header_fmt = base_fmt + '{exc:40}'
        kwfmt = {
            'pass_fail_char' : '*',
            'test_name' : 'TEST NAME',
            'test_type' : 'TYPE',
            'exc' : 'EXCEPTION INFO (AND END OF TRACEBACK)',
            }
        print(header_fmt.format(**kwfmt))
        for key in kwfmt.keys():
            kwfmt[key] = '-' * len(kwfmt[key])
        print(header_fmt.format(**kwfmt))

        # For each line in log
        with open(symlink_name) as log:
            for row in csv.reader(log):
                test_name, test_type, exc_name, filename, line_num, func_name, exc_msg = row
                test_type = testing.test_type_short_name(test_type)
                if exc_name:
                    pass_fail_char = 'F'
                    if exc_name == testing.TestSkippedException.__name__:
                        pass_fail_char = 's'
                    fmt = base_fmt + '{exc_name}:{exc_msg}'
                    if filename or line_num or func_name:
                        spacer = base_fmt.format( pass_fail_char='', test_name='', test_type='')
                        fmt += '\n' + spacer + '{filename}:{line_num}:{func_name}()'
                else:
                    pass_fail_char = '-'
                    fmt = base_fmt + 'PASS'

                print(fmt.format(
                    pass_fail_char=pass_fail_char,
                    test_name=test_name,
                    test_type=test_type,
                    exc_name=exc_name,
                    filename=filename,
                    line_num=line_num,
                    func_name=func_name,
                    exc_msg=exc_msg,
                    ))
        print(header_fmt.format(**kwfmt))
        print('* First column is [F]ail, [s]kip, or "-" for pass')

logger = None

def create():
    global logger
    logger = TestLogger()

def logger_func_factory(method_name):
    def func(*args, **kwargs):
        global logger
        method = getattr(logger, method_name)
        method(*args, **kwargs)
    return func

method_names = [
    'write',
    'close',
    'print_summary',
    'print_result_table',
    'keyboard_interrupt'
    ]
for method_name in  method_names:
    globals()[method_name] = logger_func_factory(method_name)

<commit_msg>testing - Better output formatting (removed a ':')<commit_after>from collections import defaultdict
from os.path import expanduser
import traceback
import time
import os
import csv
import testing

class TestLogger:
    def __init__(self):
        log_dir, symlink_name, log_file_name = self._log_name()
        print('Creating new log:', log_file_name)
        os.makedirs(log_dir, exist_ok=True)
        try:
            os.remove(symlink_name)
        except: pass
        os.symlink(log_file_name, symlink_name)
        self.log_file = open(symlink_name, 'w')
        self.log = csv.writer(self.log_file, delimiter=',', quotechar='"')

    def _log_name(self):
        log_dir = expanduser('~/rstem_logs/')
        symlink_name = log_dir + 'testlog.txt'
        log_file_name = 'testlog.' + str(int(time.time())) + '.txt'
        return (log_dir, symlink_name, log_file_name)

    def write(self, test, test_type, exc):
        filename = line_num = func_name = line_text = exc_name = exc_msg = ''
        if exc:
            extracted_tb = traceback.extract_tb(exc.__traceback__)
            try:
                filename, line_num, func_name, line_text = list(extracted_tb[-1])
            except IndexError:
                pass
            exc_name = type(exc).__name__
            exc_msg = str(exc)

        test_func_name = str(test.__name__) if test else ""

        self.log.writerow([
            test_func_name,
            test_type,
            exc_name,
            filename,
            line_num,
            func_name,
            exc_msg
            ])

    def keyboard_interrupt(self):
        self.write(
            None,
            None,
            KeyboardInterrupt())

    def close(self):
        self.log_file.close()

    def print_summary(self):
        log_dir, symlink_name, log_file_name = self._log_name()

        # For each line in log

        with open(symlink_name) as log:
            class PassCounts:
                passed = 0
                failed = 0
                skipped = 0
                total = 0
            pass_counts_per_test_type = {}
            for t in testing.ordered_test_types:
                pass_counts_per_test_type[t] = PassCounts()
            for row in csv.reader(log):
                test_name, test_type, exc_name, filename, line_num, func_name, exc_msg = row
                if test_type:
                    if not exc_name:
                        pass_counts_per_test_type[test_type].passed += 1
                    elif exc_name == testing.TestSkippedException.__name__:
                        pass_counts_per_test_type[test_type].skipped += 1
                    else:
                        pass_counts_per_test_type[test_type].failed += 1
                    pass_counts_per_test_type[test_type].total += 1
            print("TEST SUMMARY:")
            for t in testing.ordered_test_types:
                if pass_counts_per_test_type[t].total > 0:
                    if pass_counts_per_test_type[t].passed == pass_counts_per_test_type[t].total:
                        print('\t{}: PASSED ALL TESTS ({})'.format(
                            t,
                            pass_counts_per_test_type[t].total,
                            ))
                    elif pass_counts_per_test_type[t].skipped == 0:
                        print('\t{}: passed {} of {}'.format(
                            t,
                            pass_counts_per_test_type[t].passed,
                            pass_counts_per_test_type[t].total,
                            ))
                    else:
                        print('\t{}: passed {} of {} (skipped {})'.format(
                            t,
                            pass_counts_per_test_type[t].passed,
                            pass_counts_per_test_type[t].total,
                            pass_counts_per_test_type[t].skipped,
                            ))
                expected_total = 0
                expected_total += pass_counts_per_test_type[t].passed
                expected_total += pass_counts_per_test_type[t].failed
                expected_total += pass_counts_per_test_type[t].skipped
                if expected_total != pass_counts_per_test_type[t].total:
                    print('WARNING: Counts don\'t add to total')

    def print_result_table(self):
        log_dir, symlink_name, log_file_name = self._log_name()
        base_fmt = '{pass_fail_char:2}{test_type:5}{test_name:30}'

        # Header - 2 lines: line of labels, then line of "-" for each label.
        header_fmt = base_fmt + '{exc:40}'
        kwfmt = {
            'pass_fail_char' : '*',
            'test_name' : 'TEST NAME',
            'test_type' : 'TYPE',
            'exc' : 'EXCEPTION INFO (AND END OF TRACEBACK)',
            }
        print(header_fmt.format(**kwfmt))
        for key in kwfmt.keys():
            kwfmt[key] = '-' * len(kwfmt[key])
        print(header_fmt.format(**kwfmt))

        # For each line in log
        with open(symlink_name) as log:
            for row in csv.reader(log):
                test_name, test_type, exc_name, filename, line_num, func_name, exc_msg = row
                test_type = testing.test_type_short_name(test_type)
                if exc_name:
                    pass_fail_char = 'F'
                    if exc_name == testing.TestSkippedException.__name__:
                        pass_fail_char = 's'
                    if exc_msg:
                        fmt = base_fmt + '{exc_name}:{exc_msg}'
                    else:
                        fmt = base_fmt + '{exc_name}'
                    if filename or line_num or func_name:
                        spacer = base_fmt.format( pass_fail_char='', test_name='', test_type='')
                        fmt += '\n' + spacer + '{filename}:{line_num}:{func_name}()'
                else:
                    pass_fail_char = '-'
                    fmt = base_fmt + 'PASS'

                print(fmt.format(
                    pass_fail_char=pass_fail_char,
                    test_name=test_name,
                    test_type=test_type,
                    exc_name=exc_name,
                    filename=filename,
                    line_num=line_num,
                    func_name=func_name,
                    exc_msg=exc_msg,
                    ))
        print(header_fmt.format(**kwfmt))
        print('* First column is [F]ail, [s]kip, or "-" for pass')

logger = None

def create():
    global logger
    logger = TestLogger()

def logger_func_factory(method_name):
    def func(*args, **kwargs):
        global logger
        method = getattr(logger, method_name)
        method(*args, **kwargs)
    return func

method_names = [
    'write',
    'close',
    'print_summary',
    'print_result_table',
    'keyboard_interrupt'
    ]
for method_name in  method_names:
    globals()[method_name] = logger_func_factory(method_name)

<|endoftext|>


<commit_before># scripts to compare fit files from fitacf and fitlomb

# generate rawacf file with known targets
# process file with fitacf and fitlomb
# compare output..

import os
import sys
import pdb
import datetime
import h5py

from multiprocessing import Manager
from rawacf_generator import rawacf_record
from superdarn_tools import *

sys.path.append('../SuperDARN_FitLOMB')
from pydarncuda_fitlomb import generate_fitlomb
SANDBOX = '/home/kleinjt/repos/SuperDARN_pydmap_write/sandbox'
ACF_NAME = '20150101.0000.00.tst'
RAWACF_EXT = '.rawacf'
FITACF_EXT = '.fitacf'
RADAR = 'tst'
RECORDTIME = datetime.datetime(2015, 1, 1, 0, 0)
DEF_NOISE = 1

class target(object):
    # defines a target a range rangekm with velocity and width in meters per second
    def __init__(self, rangegate = 0, velocity = 0, width = 0, power = 0, v_e = 0, w_e = 0, nlag = 0):
        self.rangegate = rangegate
        self.velocity = velocity
        self.width = width
        self.power = power
        self.v_e = v_e
        self.w_e = w_e
        self.nlag = nlag

# synthesizes a rawacf file with velocity and spectral width at gates rgates
def generate_rawacf(rawacfname, targets = [], noise = DEF_NOISE):
    rawacf = rawacf_record(filename = rawacfname)

    for target in targets:
        rawacf.addTarget(target.rangegate, target.power, target.velocity, target.width)

    rawacf.generateScatter()
    rawacf.applyNoise(noise)
    rawacf.setTime(RECORDTIME)
    rawacf.write()
    rawacf.close()

# generates a fitacf file fitacfname from rawacf file rawacfname
def generate_fitacf(rawacfname, fitacfname):
    cmd = 'make_fit -new {} > {}'.format(rawacfname, fitacfname)
    os.system(cmd)

# generates a fitlomb file fitacfname from rawacf file rawacfname
def generate_fitlomb(rawacfname):
    os.environ['DAVIT_LOCALDIR'] = SANDBOX
    os.environ['DAVIT_DIRFORMAT'] = '%(dirtree)s/'
    radar = RADAR
    manager = Manager()
    lock = manager.Lock()

    stime = RECORDTIME
    etime = None

    record = ((stime, etime, radar, lock))
    generate_fitlomb(record)

# parse fitacf, returns a list of targets at ranges rgates
def parse_fitacf(fitacfname, rgates):
    dumpstr = get_dmapdumpstring()
    scandata = parse_dmapdumpstring(dumpstr)
    targets = []
    for rgate in rgates:
        if rgate in scandata['slist']:
            sidx = np.nonzero(scandata['slist'] == rgate)[0][0]
            v = scandata['v'][sidx]
            w_l = scandata['w_l'][sidx]
            p_l = scandata['p_l'][sidx]
            v_e = scandata['v_e'][sidx]
            w_l_e = scandata['w_l_e'][sidx]
            nlag = scandata['nlag'][sidx]

            targets.append(target(rangegate = rgate, velocity = v, width = w_l, power = p_l, v_e = v_e, w_e = w_l_e, nlag = nlag))

        else:
            print('scatter not found on rage gate {}'.format(rgate))
            targets.append(target(rangegate = rgate, nlag = 0))

    return targets

# parse fitlomb, return a list of targets at ranges rgates
def parse_fitlomb(fitlombname, rgates):
    h5f = h5py.File(fitlombname, 'r')
    beamgrp = h5f[h5f.keys()[1]]
    targets = []

    v = h5f[beamgrp]['v']
    v_e = h5f[beamgrp]['v_e']
    p_l = h5f[beamgrp]['p_l']
    w_l = h5f[beamgrp]['w_l']
    w_l_e = h5f[beamgrp]['w_l_e']
    nlag = h5f[beamgrp]['nlag']

    for rgate in rgates:
        targets.append(target(rangegate = rgate, velocity = v[rgate], width = w_l[rgate], power = p_l[rgate], v_e = v_e[rgate], w_e = w_l_e[rgate], nlag = nlag))

    return targets

def main():
    rawacf_name = SANDBOX + '/' + ACF_NAME + RAWACF_EXT
    fitacf_name = SANDBOX + '/' + ACF_NAME + FITACF_EXT
    synthetic_targets = []
    t = target(rangegate = 5, velocity = 500, width = 200, power = 1)
    synthetic_targets.append(t)

    gates = [t.rangegate for t in synthetic_targets]

    generate_rawacf(rawacf_name, targets = synthetic_targets, noise = .1)
    generate_fitacf(rawacf_name, fitacf_name)
    generate_fitlomb(rawacf_name)
    fitacf_targets = parse_fitacf(fitacf_name, gates)
    #fitlomb_targets = parse_fitlomb(fitlomb_name, gates)

    pdb.set_trace()

if __name__ == '__main__':
    main()
<commit_msg>fixed fitlomb generation..<commit_after># scripts to compare fit files from fitacf and fitlomb

# generate rawacf file with known targets
# process file with fitacf and fitlomb
# compare output..

import os
import sys
import pdb
import datetime
import h5py
import pydarn.sdio as sdio

from multiprocessing import Manager
from rawacf_generator import rawacf_record
from superdarn_tools import *

sys.path.append('../SuperDARN_FitLOMB')
import lagstate
from cuda_bayes import BayesGPU
from pydarncuda_fitlomb import CULombFit

SANDBOX = '/home/radar/repos/SuperDARN_pydmap_write/sandbox'
ACF_NAME = '20150101.0000.00.tst'
RAWACF_EXT = '.rawacf'
FITACF_EXT = '.fitacf'
HDF5_EXT = '.fitlomb.hdf5'

RADAR = 'tst'
RECORDTIME = datetime.datetime(2015, 1, 1, 0, 0)
DEF_NOISE = 1

C = 3e8
MAX_TFREQ = 16e6
LOMB_PASSES = 1
NFREQS = 256
NALFS = 256
FWHM_TO_SIGMA = 2.355 # conversion of fwhm to std deviation, assuming gaussian
MAX_V = 2000 # m/s, max velocity (doppler shift) to include in lomb
MAX_W = 1200 # m/s, max spectral width to include in lomb

class target(object):
    # defines a target a range rangekm with velocity and width in meters per second
    def __init__(self, rangegate = 0, velocity = 0, width = 0, power = 0, v_e = 0, w_e = 0, nlag = 0):
        self.rangegate = rangegate
        self.velocity = velocity
        self.width = width
        self.power = power
        self.v_e = v_e
        self.w_e = w_e
        self.nlag = nlag

# synthesizes a rawacf file with velocity and spectral width at gates rgates
def generate_rawacf(rawacfname, targets = [], noise = DEF_NOISE):
    rawacf = rawacf_record(filename = rawacfname)

    for target in targets:
        rawacf.addTarget(target.rangegate, target.power, target.velocity, target.width)

    rawacf.generateScatter()
    rawacf.applyNoise(noise)
    rawacf.setTime(RECORDTIME)
    rawacf.write()
    rawacf.close()

# generates a fitacf file fitacfname from rawacf file rawacfname
def generate_fitacf(rawacfname, fitacfname):
    cmd = 'make_fit -new {} > {}'.format(rawacfname, fitacfname)
    os.system(cmd)

# generates a fitlomb file fitacfname from rawacf file rawacfname
def generate_fitlomb(rawacfname, fitlombname):
    os.environ['DAVIT_LOCALDIR'] = SANDBOX
    os.environ['DAVIT_DIRFORMAT'] = '%(dirtree)s/'
    radar = RADAR
    manager = Manager()
    lock = manager.Lock()

    stime = RECORDTIME
    etime = None

    hdf5file = h5py.File(fitlombname, 'w')

    myPtr = sdio.radDataOpen(stime,radar,eTime=etime,channel=None,bmnum=None,cp=None,fileType='rawacf',filtered=False, src='local', noCache = True)
    drec = sdio.radDataReadRec(myPtr)

    amax = np.ceil((np.pi * 2 * MAX_TFREQ * MAX_W) / C)
    fmax = np.ceil(MAX_V * 2 * MAX_TFREQ / C)
    freqs = np.linspace(-fmax,fmax, NFREQS)
    alfs = np.linspace(0, amax, NALFS)

    fit = CULombFit(drec)
    gpu_lambda = BayesGPU(fit.lags, freqs, alfs, fit.nrang, LAMBDA_FIT)
    txlag_cache = lagstate.good_lags_txsamples(fit)
    fit.SetBadlags(txlag_cache = txlag_cache)
    fit.CudaProcessPulse(gpu_lambda)
    fit.CudaCopyPeaks(gpu_lambda)
    fit.WriteLSSFit(hdf5file)
    hdf5file.close()

# parse fitacf, returns a list of targets at ranges rgates
def parse_fitacf(fitacfname, rgates):
    dumpstr = get_dmapdumpstring()
    scandata = parse_dmapdumpstring(dumpstr)
    targets = []
    for rgate in rgates:
        if rgate in scandata['slist']:
            sidx = np.nonzero(scandata['slist'] == rgate)[0][0]
            v = scandata['v'][sidx]
            w_l = scandata['w_l'][sidx]
            p_l = scandata['p_l'][sidx]
            v_e = scandata['v_e'][sidx]
            w_l_e = scandata['w_l_e'][sidx]
            nlag = scandata['nlag'][sidx]

            targets.append(target(rangegate = rgate, velocity = v, width = w_l, power = p_l, v_e = v_e, w_e = w_l_e, nlag = nlag))

        else:
            print('scatter not found on range gate {}'.format(rgate))
            targets.append(target(rangegate = rgate, nlag = 0))

    return targets

# parse fitlomb, return a list of targets at ranges rgates
def parse_fitlomb(fitlombname, rgates):
    h5f = h5py.File(fitlombname, 'r')
    beamgrp = h5f[h5f.keys()[0]]
    targets = []

    v = beamgrp['v'][...]
    v_e = beamgrp['v_e'][...]
    p_l =  beamgrp['p_l'][...]
    w_l =  beamgrp['w_l'][...]
    w_l_e = beamgrp['w_l_e'][...]
    nlag = beamgrp['nlag'][...]

    for rgate in rgates:
        targets.append(target(rangegate = rgate, velocity = v[rgate], width = w_l[rgate], power = p_l[rgate], v_e = v_e[rgate], w_e = w_l_e[rgate], nlag = nlag))

    return targets

def main():
    print 'finished importing...'
    rawacf_name = SANDBOX + '/' + ACF_NAME + RAWACF_EXT
    fitacf_name = SANDBOX + '/' + ACF_NAME + FITACF_EXT
    fitlomb_name = SANDBOX + '/' + ACF_NAME + HDF5_EXT

    print 'creating targets...'
    synthetic_targets = []
    t = target(rangegate = 5, velocity = 500, width = 200, power = 1)
    synthetic_targets.append(t)

    gates = [t.rangegate for t in synthetic_targets]

    print 'creating rawacf...'
    generate_rawacf(rawacf_name, targets = synthetic_targets, noise = .1)

    print 'creating fitacf...'
    generate_fitacf(rawacf_name, fitacf_name)

    print 'creating fitlomb...'
    generate_fitlomb(rawacf_name, fitlomb_name)

    fitacf_targets = parse_fitacf(fitacf_name, gates)
    fitlomb_targets = parse_fitlomb(fitlomb_name, gates)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>""" Extens functools"""

import functools

def compose(*args):
    """
    :param args: a list of functions
    :return: composition of the functions
    """
    def compose2(f1, f2):
        def composed(*args_c, **kwargs_c):
            return f1(f2(*args_c, **kwargs_c))
        return composed

    return functools.reduce(compose2, args)

<commit_msg>nicer import<commit_after>""" Extens functools"""

# Allows user to write import functools_ext to get everything from functools
from functools import *

def compose(*args):
    """
    :param args: a list of functions
    :return: composition of the functions
    """
    def compose2(f1, f2):
        def composed(*args_c, **kwargs_c):
            return f1(f2(*args_c, **kwargs_c))
        return composed

    return reduce(compose2, args)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Author: trummerjo
# Module: MSLHttpRequestHandler
# Created on: 26.01.2017
# License: MIT https://goo.gl/5bMj3H

"""Handles & translates requests from Inputstream to Netflix"""

import base64
import BaseHTTPServer
from urlparse import urlparse, parse_qs

from SocketServer import TCPServer
from resources.lib.MSL import MSL

class MSLHttpRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handles & translates requests from Inputstream to Netflix"""

    # pylint: disable=invalid-name
    def do_HEAD(self):
        """Answers head requests with a success code"""
        self.send_response(200)

    # pylint: disable=invalid-name
    def do_POST(self):
        """Loads the licence for the requested resource"""
        length = int(self.headers.get('content-length'))
        post = self.rfile.read(length)
        data = post.split('!')
        if len(data) is 2:
            challenge = data[0]
            sid = base64.standard_b64decode(data[1])
            b64license = self.server.msl_handler.get_license(challenge, sid)
            if b64license is not '':
                self.send_response(200)
                self.end_headers()
                self.wfile.write(base64.standard_b64decode(b64license))
                self.finish()
            else:
                self.server.nx_common.log(msg='Error getting License')
                self.send_response(400)
        else:
            self.server.nx_common.log(msg='Error in License Request')
            self.send_response(400)

    # pylint: disable=invalid-name
    def do_GET(self):
        """Loads the XML manifest for the requested resource"""
        url = urlparse(self.path)
        params = parse_qs(url.query)
        if 'id' not in params:
            self.send_response(400, 'No id')
        else:
            # Get the manifest with the given id
            dolby = (True if 'dolby' in params and
                     params['dolby'][0].lower() == 'true' else 'false')
            hevc = (True if 'hevc' in params and
                    params['hevc'][0].lower() == 'true' else 'false')
            hdr = (True if 'hdr' in params and
                    params['hdr'][0].lower() == 'true' else 'false')
            dolbyvision = (True if 'dolbyvision' in params and
                    params['dolbyvision'][0].lower() == 'true' else 'false')
            vp9 = (True if 'vp9' in params and
                    params['vp9'][0].lower() == 'true' else 'false')

            data = self.server.msl_handler.load_manifest(
                int(params['id'][0]),
                dolby, hevc, hdr, dolbyvision, vp9)

            self.send_response(200)
            self.send_header('Content-type', 'application/xml')
            self.end_headers()
            self.wfile.write(data)

    def log_message(self, *args):
        """Disable the BaseHTTPServer Log"""
        pass

##################################

class MSLTCPServer(TCPServer):
    """Override TCPServer to allow usage of shared members"""

    def __init__(self, server_address, nx_common):
        """Initialization of MSLTCPServer"""
        nx_common.log(msg='Constructing MSLTCPServer')
        self.nx_common = nx_common
        self.msl_handler = MSL(nx_common)
        TCPServer.__init__(self, server_address, MSLHttpRequestHandler)

    def reset_msl_data(self):
        """Initialization of MSLTCPServerResets MSL data (perform handshake)"""
        self.msl_handler.perform_key_handshake()
<commit_msg>Fix parameter values for stream play<commit_after># -*- coding: utf-8 -*-
# Author: trummerjo
# Module: MSLHttpRequestHandler
# Created on: 26.01.2017
# License: MIT https://goo.gl/5bMj3H

"""Handles & translates requests from Inputstream to Netflix"""

import base64
import BaseHTTPServer
from urlparse import urlparse, parse_qs

from SocketServer import TCPServer
from resources.lib.MSL import MSL

class MSLHttpRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handles & translates requests from Inputstream to Netflix"""

    # pylint: disable=invalid-name
    def do_HEAD(self):
        """Answers head requests with a success code"""
        self.send_response(200)

    # pylint: disable=invalid-name
    def do_POST(self):
        """Loads the licence for the requested resource"""
        length = int(self.headers.get('content-length'))
        post = self.rfile.read(length)
        data = post.split('!')
        if len(data) is 2:
            challenge = data[0]
            sid = base64.standard_b64decode(data[1])
            b64license = self.server.msl_handler.get_license(challenge, sid)
            if b64license is not '':
                self.send_response(200)
                self.end_headers()
                self.wfile.write(base64.standard_b64decode(b64license))
                self.finish()
            else:
                self.server.nx_common.log(msg='Error getting License')
                self.send_response(400)
        else:
            self.server.nx_common.log(msg='Error in License Request')
            self.send_response(400)

    # pylint: disable=invalid-name
    def do_GET(self):
        """Loads the XML manifest for the requested resource"""
        url = urlparse(self.path)
        params = parse_qs(url.query)
        if 'id' not in params:
            self.send_response(400, 'No id')
        else:
            # Get the manifest with the given id
            dolby = (True if 'dolby' in params and
                     params['dolby'][0].lower() == 'true' else False)
            hevc = (True if 'hevc' in params and
                    params['hevc'][0].lower() == 'true' else False)
            hdr = (True if 'hdr' in params and
                    params['hdr'][0].lower() == 'true' else False)
            dolbyvision = (True if 'dolbyvision' in params and
                    params['dolbyvision'][0].lower() == 'true' else False)
            vp9 = (True if 'vp9' in params and
                    params['vp9'][0].lower() == 'true' else False)

            data = self.server.msl_handler.load_manifest(
                int(params['id'][0]),
                dolby, hevc, hdr, dolbyvision, vp9)

            self.send_response(200)
            self.send_header('Content-type', 'application/xml')
            self.end_headers()
            self.wfile.write(data)

    def log_message(self, *args):
        """Disable the BaseHTTPServer Log"""
        pass

##################################

class MSLTCPServer(TCPServer):
    """Override TCPServer to allow usage of shared members"""

    def __init__(self, server_address, nx_common):
        """Initialization of MSLTCPServer"""
        nx_common.log(msg='Constructing MSLTCPServer')
        self.nx_common = nx_common
        self.msl_handler = MSL(nx_common)
        TCPServer.__init__(self, server_address, MSLHttpRequestHandler)

    def reset_msl_data(self):
        """Initialization of MSLTCPServerResets MSL data (perform handshake)"""
        self.msl_handler.perform_key_handshake()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collections
import contextlib
import json

import yaml

from .atomic_template import AtomicTemplate

class Template(object):

    def __init__(self):
        self._templates = []

    def merge(self, template):
        self._templates.append(template)

    def to_json(self, indent=4, sort_keys=True, separators=(',', ': ')):
        return json.dumps(
            self.template, indent=indent,
            sort_keys=sort_keys, separators=separators
        )

    def to_yaml(self, default_flow_style=False):
        return yaml.safe_dump(
            self.template, default_flow_style=default_flow_style
        )

    @property
    def _parameters(self):
        parameters = {}
        for template in self._templates:
            parameters.update(template._parameters)
        return parameters

    @property
    def _resources(self):
        resources = {}
        for template in self._templates:
            resources.update(template._resources)
        return resources

    @property
    def template(self):
        with unique_atom_names(self._templates):
            template = {
                "Parameters": self._parameters,
                "Resources": self._resources
            }
        return template

def flatten(templates):
    flattened_templates = []
    for template in templates:
        if isinstance(template, AtomicTemplate):
            flattened_templates.append(template)
        else:
            # TODO: I'd like templates to be private, but accessing it here
            # isn't great.
            flattened_templates.extend(flatten(template._templates))
    return flattened_templates

@contextlib.contextmanager
def unique_atom_names(templates):
    # TODO: look into optimising this.
    flattened_templates = flatten(templates)
    old_names = {}
    name_counter = collections.Counter()
    for template in flattened_templates:
        name_counter.update([template.name])
        name_count = name_counter[template.name]
        if name_count > 1:
            temp_name = "".join([template.name, str(name_count - 1)])
            old_names[temp_name] = template.name
            template.name = temp_name
    yield
    for template in flattened_templates:
        if template.name in old_names:
            template.name = old_names[template.name]
<commit_msg>Add outputs to template<commit_after># -*- coding: utf-8 -*-

import collections
import contextlib
import json

import yaml

from .atomic_template import AtomicTemplate

class Template(object):

    def __init__(self):
        self._templates = []

    def merge(self, template):
        self._templates.append(template)

    def to_json(self, indent=4, sort_keys=True, separators=(',', ': ')):
        return json.dumps(
            self.template, indent=indent,
            sort_keys=sort_keys, separators=separators
        )

    def to_yaml(self, default_flow_style=False):
        return yaml.safe_dump(
            self.template, default_flow_style=default_flow_style
        )

    @property
    def _outputs(self):
        outputs = {}
        for template in self._templates:
            outputs.update(template._outputs)
        return outputs

    @property
    def _parameters(self):
        parameters = {}
        for template in self._templates:
            parameters.update(template._parameters)
        return parameters

    @property
    def _resources(self):
        resources = {}
        for template in self._templates:
            resources.update(template._resources)
        return resources

    @property
    def template(self):
        with unique_atom_names(self._templates):
            template = {
                "Parameters": self._parameters,
                "Resources": self._resources,
                "Outputs": self._outputs
            }
        return template

def flatten(templates):
    flattened_templates = []
    for template in templates:
        if isinstance(template, AtomicTemplate):
            flattened_templates.append(template)
        else:
            # TODO: I'd like templates to be private, but accessing it here
            # isn't great.
            flattened_templates.extend(flatten(template._templates))
    return flattened_templates

@contextlib.contextmanager
def unique_atom_names(templates):
    # TODO: look into optimising this.
    flattened_templates = flatten(templates)
    old_names = {}
    name_counter = collections.Counter()
    for template in flattened_templates:
        name_counter.update([template.name])
        name_count = name_counter[template.name]
        if name_count > 1:
            temp_name = "".join([template.name, str(name_count - 1)])
            old_names[temp_name] = template.name
            template.name = temp_name
    yield
    for template in flattened_templates:
        if template.name in old_names:
            template.name = old_names[template.name]
<|endoftext|>


<commit_before>import discord
import asyncio
import random
import json
import time
import requests
import git
import subprocess
import sys
import os
import logging
import aiohttp
import base64
from datetime import datetime
import modules.commands as commands
import modules.misc as misc
import modules.pokemon as pokemon
import modules.admin as admin
import modules.embed_commands as embed_commands
import modules.google_search as google_search
import modules.owner as owner
import modules.random as random_cmds
import modules.voice as voice

from google import search

if not discord.opus.is_loaded():
    try:
        discord.opus.load_opus('opus')
    except OSError:
        try:
            discord.opus.load_opus('libopus')
        except OSError:
            print('ERORR, Voice Will be disabled do to opus loading error')

if discord.opus.is_loaded():
    print('Voice Loaded!')

logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(asctime)s: %(message)s')
try:
    with open("settings.json", "r") as infile:
        settings = json.loads(infile.read())
except:
    print('No Settings.json file located! please run setup.py first!')
    sys.exit()

with open("database/leave_toggle.json","r") as infile:
    leavemsg = json.loads(infile.read())

with open("database/quoteweenie.json","r") as infile:
    Quotes_All = json.loads(infile.read())

with open("database/adminweenie.json","r") as infile:
    admin = json.loads(infile.read())

with open("database/storage.json", "r") as infile:
    storage = json.loads(infile.read())

with open("database/storage2.json", "r") as infile:
    storage2 = json.loads(infile.read())

class Weenie(discord.Client):
    def __init__(self, *args, **kwargs):
        self.timer = 0
        self.suspend = False
        self.repl = False
        self.voiceMap = {}
        self.voiceQ = {}
        super().__init__(*args, **kwargs)

status = {
    'online': 'Online',
    'offline': 'Offline',
    'idle': 'Idle',
    'dnd': 'Do Not Disturb'
}

x33 = '%m-%d-%Y'
client = Weenie()
gamet = discord.Game(name='With Computers')
def bdel(s, r): return (s[len(r):] if s.startswith(r) else s)

@client.event
async def on_ready():
    client.bot_info = await client.application_info()
    print('Bot owner:\n' + client.bot_info.owner.id)
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('<Servers>')
    for server in client.servers:
        print(server)
    print('------')
    await client.change_presence(game=gamet, status='online', afk=False)
    if client.bot_info.owner.id not in admin:
        admin.append(client.bot_info.owner.id)

@client.event
async def on_member_ban(member):
    if leavemsg[member.server.id] == 1:
        for i in channel.server.channels:
            if i.name == 'logs':
                    await client.send_message(i, '{} Has just been banned from the server :( Bye!'.format(member.name))
        try:
            await client.send_message(member.server.default_channel, '{} Has just been banned from the server :( Bye!'.format(member.name))
        except:
            pass

@client.event
async def on_server_join(server):
    await client.send_message(server, 'Hello! I am WeenieBot, your robot friend, here to help you with your needs on this server! type `!help` to see what I can do for you!')

@client.event
async def on_channel_delete(channel):
    for i in channel.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            cd_details = discord.Embed(title='Channel Deleted!', description='', colour=int(rr, 16))
            cd_details.add_field(name='Channel Name:', value=channel.name, inline=True)
            cd_details.add_field(name='Channel ID:', value=channel.id, inline=True)
            cd_details.add_field(name='Channel Topic:', value=channel.topic, inline=True)
            cd_details.set_author(name='Removed channel...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=cd_details)

@client.event
async def on_channel_create(channel):
    for i in channel.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            cc_details = discord.Embed(title='Channel Created!', description='', colour=int(rr, 16))
            cc_details.add_field(name='Channel Name:', value=channel.name, inline=True)
            cc_details.add_field(name='Channel ID:', value=channel.id, inline=True)
            cc_details.add_field(name='Channel Topic:', value=channel.topic, inline=True)
            cc_details.set_author(name='New channel...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=cc_details)

@client.event
async def on_message_edit(before, after):
    for i in before.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            em_details = discord.Embed(title='Message Edited!', description='', colour=int(rr, 16))
            em_details.add_field(name='Author:', value=before.author, inline=True)
            em_details.add_field(name='Before:', value=before.content, inline=True)
            em_details.add_field(name='After:', value=after.content, inline=True)
            em_details.set_author(name='Edited message...', icon_url=before.author.avatar_url)
            await client.send_message(i, embed=em_details)

@client.event
async def on_message_delete(message):
    for i in message.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            md_details = discord.Embed(title='Message Deleted!', description='', colour=int(rr, 16))
            md_details.add_field(name='Author:', value=message.author, inline=True)
            md_details.add_field(name='Message:', value=message.content, inline=True)
            md_details.set_author(name='Deleted message...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=md_details)

@client.event
async def on_member_join(member):
    with open("database/storage2.json", "r") as infile:
        storage2 = json.loads(infile.read())
    if member.server.id not in storage2:
        storage2[member.server.id] = 'message0'
    with open("database/storage2.json", "w+") as outfile:
        outfile.write(json.dumps(storage2))

    if storage2[member.server.id] == 'message0':
        try:
            for i in member.server.channels:
                if i.name == 'logs':
                    await client.send_message(i, "{0.mention} has joined {0.server.name} give them a warm welcome!".format(member))
            try:
                await client.send_message(member.server.default_channel,"{0.name} has joined {0.server.name} give them a warm welcome!".format(member))
            except discord.Forbidden:
                print('Couldn\'t welcome {} in server {} due to perms error.'.format(member, member.server))
        except discord.Forbidden:
            print('Couldn\'t welcome {} in server {} due to perms error.'.format(member, member.server))

@client.event
async def on_member_remove(member):
    if leavemsg[member.server.id] == 1:
        for i in member.server.channels:
            if i.name == 'logs':
                r = lambda: random.randint(0,255)
                rr = ('0x%02X%02X%02X' % (r(),r(),r()))
                rm_details = discord.Embed(title='Member left!', colour=int(rr, 16))
                rm_details.add_field(name='Username:', value=member.name, inline=True)
                rm_details.add_field(name='Nick:', value=member.nick, inline =True)
                rm_details.add_field(name='Joined Server:', value=member.joined_at.strftime(x33), inline =True)
                rm_details.add_field(name='Account Created:', value=member.created_at.strftime(x33), inline=True)
                rm_details.set_author(name='We lost a member :(', icon_url=member.server.get_member_named(username).avatar_url)
                await client.send_message(i, embed=rm_details)
        try:
            await client.send_message(member.server.default_channel, "{0.name} has left {0.server.name} we will miss you!".format(member))
        except:
            pass
@client.event
async def on_message(message):
    try:
        if message.server.id not in client.voiceQ:
            client.voiceQ[message.server.id] = Voice = voice.Voice(client)
    except:
        print('Errored: Most likely due to Private DM\'s')

    try:
        for channel in message.server.channels:
            if str(channel.type) == 'voice':
                client.voiceMap[channel.name] = channel.id
    except:
        pass
    with open("database/prefixMap.json", "r") as infile:
        prefixMap = json.loads(infile.read())

    try:
        if message.server.id not in storage:
            storage[message.server.id] = 'broadcast0'
    except:
        pass
    with open("database/storage.json", "w+") as outfile:
        outfile.write(json.dumps(storage))
    try:
        if message.server.id not in leavemsg.keys():
            leavemsg[message.server.id] = '1'
    except:
        pass
    with open("database/leave-toggle.json", "w+") as outfile:
        outfile.write(json.dumps(leavemsg))

    try:
        if message.server.id not in prefixMap:
            prefixMap[message.server.id] = '!'
            client.pfix = prefixMap[message.server.id]
        else:
            client.pfix = prefixMap[message.server.id]
    except:
        print('Errored: Most likely due to Private DM\'s')

    if message.content.startswith(client.pfix)  and client.suspend == False and message.author.bot is False:
        try:
            if message.content == client.pfix + 'quote' :
                await misc.rand_quote(message, client)
            elif message.content.split(' ')[0] == client.pfix + 'quote':
                await misc.quote_logic(message, client)
            else:
                cmd = bdel(message.content.lower(), client.pfix)
                cmd = cmd.split(' ')
                for i in commands.cmdDict.keys():
                    if cmd[0] in i:
                        await commands.cmdDict[i](message, client)
        except KeyError as e:
            print(str(e) + ' No command')

    if message.content == client.pfix + 'resume' and client.suspend == True:
        await commands.resume_logic(message, client)

    if message.content.startswith(client.pfix + 'skip'):
        await client.voiceQ[message.server.id].skip(message, client)

    if message.content.startswith(client.pfix + 'join'):
        await client.voiceQ[message.server.id].join(message, client)

    if message.content.startswith(client.pfix + 'play'):
        await client.voiceQ[message.server.id].play(message, client)

    if message.content.startswith(client.pfix + 'stop'):
        await client.voiceQ[message.server.id].stop(message, client)

    if message.content.startswith(client.pfix + 'pause'):
        await client.voiceQ[message.server.id].pause(message, client)

    if message.content.startswith(client.pfix + 'unpause'):
        await client.voiceQ[message.server.id].mresume(message, client)

    if message.content.startswith(client.pfix + 'disconnect'):
        await client.voiceQ[message.server.id].disconnect(message, client)

    if message.content.startswith(client.pfix + 'volume'):
        await client.voiceQ[message.server.id].volume(message, client)

    if message.content.startswith(client.pfix + 'nowplaying'):
        await client.voiceQ[message.server.id].playing(message, client)

    if message.content.lower().startswith('weeniebot'):
                await commands.cleverbot_logic(message, client)
    if message.content == "!prefix":
        await misc.get_prefix_logic(message, client)

    if message.content == client.pfix + 'turtles':
        await client.send_message(message.channel, 'https://www.youtube.com/watch?v=o4PBYRN-ndI')

client.run(settings['token'])
<commit_msg>trying to source the leave message problem<commit_after>import discord
import asyncio
import random
import json
import time
import requests
import git
import subprocess
import sys
import os
import logging
import aiohttp
import base64
from datetime import datetime
import modules.commands as commands
import modules.misc as misc
import modules.pokemon as pokemon
import modules.admin as admin
import modules.embed_commands as embed_commands
import modules.google_search as google_search
import modules.owner as owner
import modules.random as random_cmds
import modules.voice as voice

from google import search

if not discord.opus.is_loaded():
    try:
        discord.opus.load_opus('opus')
    except OSError:
        try:
            discord.opus.load_opus('libopus')
        except OSError:
            print('ERORR, Voice Will be disabled do to opus loading error')

if discord.opus.is_loaded():
    print('Voice Loaded!')

logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(asctime)s: %(message)s')
try:
    with open("settings.json", "r") as infile:
        settings = json.loads(infile.read())
except:
    print('No Settings.json file located! please run setup.py first!')
    sys.exit()

with open("database/leave_toggle.json","r") as infile:
    leavemsg = json.loads(infile.read())

with open("database/quoteweenie.json","r") as infile:
    Quotes_All = json.loads(infile.read())

with open("database/adminweenie.json","r") as infile:
    admin = json.loads(infile.read())

with open("database/storage.json", "r") as infile:
    storage = json.loads(infile.read())

with open("database/storage2.json", "r") as infile:
    storage2 = json.loads(infile.read())

class Weenie(discord.Client):
    def __init__(self, *args, **kwargs):
        self.timer = 0
        self.suspend = False
        self.repl = False
        self.voiceMap = {}
        self.voiceQ = {}
        super().__init__(*args, **kwargs)

status = {
    'online': 'Online',
    'offline': 'Offline',
    'idle': 'Idle',
    'dnd': 'Do Not Disturb'
}

x33 = '%m-%d-%Y'
client = Weenie()
gamet = discord.Game(name='With Computers')
def bdel(s, r): return (s[len(r):] if s.startswith(r) else s)

@client.event
async def on_ready():
    client.bot_info = await client.application_info()
    print('Bot owner:\n' + client.bot_info.owner.id)
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('<Servers>')
    for server in client.servers:
        print(server)
    print('------')
    await client.change_presence(game=gamet, status='online', afk=False)
    if client.bot_info.owner.id not in admin:
        admin.append(client.bot_info.owner.id)

@client.event
async def on_member_ban(member):
    if leavemsg[member.server.id] == 1:
        for i in channel.server.channels:
            if i.name == 'logs':
                    await client.send_message(i, '{} Has just been banned from the server :( Bye!'.format(member.name))
        try:
            await client.send_message(member.server.default_channel, '{} Has just been banned from the server :( Bye!'.format(member.name))
        except:
            pass

@client.event
async def on_server_join(server):
    await client.send_message(server, 'Hello! I am WeenieBot, your robot friend, here to help you with your needs on this server! type `!help` to see what I can do for you!')

@client.event
async def on_channel_delete(channel):
    for i in channel.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            cd_details = discord.Embed(title='Channel Deleted!', description='', colour=int(rr, 16))
            cd_details.add_field(name='Channel Name:', value=channel.name, inline=True)
            cd_details.add_field(name='Channel ID:', value=channel.id, inline=True)
            cd_details.add_field(name='Channel Topic:', value=channel.topic, inline=True)
            cd_details.set_author(name='Removed channel...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=cd_details)

@client.event
async def on_channel_create(channel):
    for i in channel.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            cc_details = discord.Embed(title='Channel Created!', description='', colour=int(rr, 16))
            cc_details.add_field(name='Channel Name:', value=channel.name, inline=True)
            cc_details.add_field(name='Channel ID:', value=channel.id, inline=True)
            cc_details.add_field(name='Channel Topic:', value=channel.topic, inline=True)
            cc_details.set_author(name='New channel...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=cc_details)

@client.event
async def on_message_edit(before, after):
    for i in before.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            em_details = discord.Embed(title='Message Edited!', description='', colour=int(rr, 16))
            em_details.add_field(name='Author:', value=before.author, inline=True)
            em_details.add_field(name='Before:', value=before.content, inline=True)
            em_details.add_field(name='After:', value=after.content, inline=True)
            em_details.set_author(name='Edited message...', icon_url=before.author.avatar_url)
            await client.send_message(i, embed=em_details)

@client.event
async def on_message_delete(message):
    for i in message.server.channels:
        if i.name == 'logs':
            r = lambda: random.randint(0,255)
            rr = ('0x%02X%02X%02X' % (r(),r(),r()))
            md_details = discord.Embed(title='Message Deleted!', description='', colour=int(rr, 16))
            md_details.add_field(name='Author:', value=message.author, inline=True)
            md_details.add_field(name='Message:', value=message.content, inline=True)
            md_details.set_author(name='Deleted message...', icon_url=message.author.avatar_url)
            await client.send_message(i, embed=md_details)

@client.event
async def on_member_join(member):
    with open("database/storage2.json", "r") as infile:
        storage2 = json.loads(infile.read())
    if member.server.id not in storage2:
        storage2[member.server.id] = 'message0'
    with open("database/storage2.json", "w+") as outfile:
        outfile.write(json.dumps(storage2))

    if storage2[member.server.id] == 'message0':
        try:
            for i in member.server.channels:
                if i.name == 'logs':
                    await client.send_message(i, "{0.mention} has joined {0.server.name} give them a warm welcome!".format(member))
            try:
                await client.send_message(member.server.default_channel,"{0.name} has joined {0.server.name} give them a warm welcome!".format(member))
            except discord.Forbidden:
                print('Couldn\'t welcome {} in server {} due to perms error.'.format(member, member.server))
        except discord.Forbidden:
            print('Couldn\'t welcome {} in server {} due to perms error.'.format(member, member.server))

@client.event
async def on_member_remove(member):
    if leavemsg[member.server.id] == 1:
        for i in member.server.channels:
            if i.name == 'logs':
                r = lambda: random.randint(0,255)
                rr = ('0x%02X%02X%02X' % (r(),r(),r()))
                rm_details = discord.Embed(title='Member left!', colour=int(rr, 16))
                rm_details.add_field(name='Username:', value=member.name, inline=True)
                rm_details.add_field(name='Nick:', value=member.nick, inline =True)
                rm_details.add_field(name='Joined Server:', value=member.joined_at.strftime(x33), inline =True)
                rm_details.add_field(name='Account Created:', value=member.created_at.strftime(x33), inline=True)
                rm_details.set_author(name='We lost a member :(', icon_url=member.server.get_member_named(username).avatar_url)
                await client.send_message(i, embed=rm_details)
        try:
            await client.send_message(member.server.default_channel, "{0.name} has left {0.server.name} we will miss you!".format(member))
        except:
            pass
@client.event
async def on_message(message):
    try:
        if message.server.id not in client.voiceQ:
            client.voiceQ[message.server.id] = Voice = voice.Voice(client)
    except:
        print('Errored: Most likely due to Private DM\'s')

    try:
        for channel in message.server.channels:
            if str(channel.type) == 'voice':
                client.voiceMap[channel.name] = channel.id
    except:
        pass
    with open("database/prefixMap.json", "r") as infile:
        prefixMap = json.loads(infile.read())

    try:
        if message.server.id not in storage:
            storage[message.server.id] = 'broadcast0'
    except:
        pass
    with open("database/storage.json", "w+") as outfile:
        outfile.write(json.dumps(storage))
    if message.server.id not in leavemsg.keys():
        leavemsg[message.server.id] = '1'
    with open("database/leave-toggle.json", "w+") as outfile:
        outfile.write(json.dumps(leavemsg))

    try:
        if message.server.id not in prefixMap:
            prefixMap[message.server.id] = '!'
            client.pfix = prefixMap[message.server.id]
        else:
            client.pfix = prefixMap[message.server.id]
    except:
        print('Errored: Most likely due to Private DM\'s')

    if message.content.startswith(client.pfix)  and client.suspend == False and message.author.bot is False:
        try:
            if message.content == client.pfix + 'quote' :
                await misc.rand_quote(message, client)
            elif message.content.split(' ')[0] == client.pfix + 'quote':
                await misc.quote_logic(message, client)
            else:
                cmd = bdel(message.content.lower(), client.pfix)
                cmd = cmd.split(' ')
                for i in commands.cmdDict.keys():
                    if cmd[0] in i:
                        await commands.cmdDict[i](message, client)
        except KeyError as e:
            print(str(e) + ' No command')

    if message.content == client.pfix + 'resume' and client.suspend == True:
        await commands.resume_logic(message, client)

    if message.content.startswith(client.pfix + 'skip'):
        await client.voiceQ[message.server.id].skip(message, client)

    if message.content.startswith(client.pfix + 'join'):
        await client.voiceQ[message.server.id].join(message, client)

    if message.content.startswith(client.pfix + 'play'):
        await client.voiceQ[message.server.id].play(message, client)

    if message.content.startswith(client.pfix + 'stop'):
        await client.voiceQ[message.server.id].stop(message, client)

    if message.content.startswith(client.pfix + 'pause'):
        await client.voiceQ[message.server.id].pause(message, client)

    if message.content.startswith(client.pfix + 'unpause'):
        await client.voiceQ[message.server.id].mresume(message, client)

    if message.content.startswith(client.pfix + 'disconnect'):
        await client.voiceQ[message.server.id].disconnect(message, client)

    if message.content.startswith(client.pfix + 'volume'):
        await client.voiceQ[message.server.id].volume(message, client)

    if message.content.startswith(client.pfix + 'nowplaying'):
        await client.voiceQ[message.server.id].playing(message, client)

    if message.content.lower().startswith('weeniebot'):
                await commands.cleverbot_logic(message, client)
    if message.content == "!prefix":
        await misc.get_prefix_logic(message, client)

    if message.content == client.pfix + 'turtles':
        await client.send_message(message.channel, 'https://www.youtube.com/watch?v=o4PBYRN-ndI')

client.run(settings['token'])
<|endoftext|>