<commit_before>from hashlib import md5, sha256

from speakerlib import niceify_number

import requests

from config import config

from fuzzywuzzy import process

class EventRecorder(object):

    def __init__(self, db):

        self.db = db
        #self.censored_words = self.db.execute('')

    def censor(*args, **kwargs):

        if not config.get("censor", False):
            return args, kwargs

        self = args[0]
        args = list(args)
        censored_words = self.db.execute("select word from badwords")
        bad_words = [row["word"] for row in censored_words]

        speech_text = kwargs.get("speech_text", None) or args[1]

        try:
            speech_text.decode("ascii")
        except UnicodeEncodeError:
            speech_text = "!random"

        speech_list = speech_text.split(" ")

        for phrase in bad_words:
            if process.extractOne(phrase, speech_list)[1] >= 90:
                speech_text = "!random"
                break

        for word in speech_list:
            if word.lower() in bad_words:
                speech_text = "!random"
                break

        if kwargs.get("speech_text", None):
            kwargs["speech_text"] = speech_text
        else:
            args[1] = speech_text

        return tuple(args)[1:], kwargs

    def post_to_slack(self, speech_text, record_utterance=False, **kwargs):
        #Skip plugins
        if speech_text[0] == "!":
            return

        if config.get("slack_url", None) and record_utterance:
            requests.post(config["slack_url"], data=speech_text)

    def record_utterance(self, speech_text, record_utterance=False, event_result=None):
        if not record_utterance or speech_text[0] == "!":
            return

        speech_text = speech_text.lower()

        sha = sha256()
        sha.update(speech_text)
        sha_hash = sha.hexdigest()

        matched_snippet = self.db.execute("SELECT votes FROM snippets where sha256=?", [sha_hash]).fetchone()

        if matched_snippet:
            votes = matched_snippet["votes"] + 1
            self.db.execute("UPDATE snippets set votes=? where sha256=?", [votes, sha_hash])
        else:
            self.db.execute("INSERT into snippets (sha256, speech_text, votes) VALUES (?, ?, 0) ", [sha_hash, speech_text])

    def record_sound_event(self, sound_name, event_result=None):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [sound_name]).fetchone()

        if matched_sound:
            votes = matched_sound["votes"] + 1

            self.db.execute("UPDATE sounds set votes=? where name=?", [votes, sound_name])

    def queue_speech_for_tweet(self, speech_text, record_utterance=False, event_result=None):

        print record_utterance
        if not record_utterance:
            return

        if speech_text[0] == "!":
            return

        speech_text = speech_text[:139].replace("@","~")

        md5_hash = md5()
        md5_hash.update(speech_text.lower())
        md5_hash = md5_hash.hexdigest()

        try:
            check_existence = self.db.execute("SELECT * FROM publish_queue where md5_hash=?", [md5_hash]).next()
        except StopIteration:
            check_existence = False

        if not check_existence:

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [speech_text, md5_hash])

    def queue_sound_for_tweet(self, name, event_result):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

        if matched_sound:
            text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
            md5_hash = md5()
            md5_hash.update(text)
            md5_hash = md5_hash.hexdigest()

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [text, md5_hash])

<commit_msg>buuuuug<commit_after>from hashlib import md5, sha256

from speakerlib import niceify_number

import requests

from config import config

from fuzzywuzzy import process

class EventRecorder(object):

    def __init__(self, db):

        self.db = db
        #self.censored_words = self.db.execute('')

    def censor(*args, **kwargs):

        if not config.get("censor", False):
            return args[1:], kwargs

        self = args[0]
        args = list(args)
        censored_words = self.db.execute("select word from badwords")
        bad_words = [row["word"] for row in censored_words]

        speech_text = kwargs.get("speech_text", None) or args[1]

        try:
            speech_text.decode("ascii")
        except UnicodeEncodeError:
            speech_text = "!random"

        speech_list = speech_text.split(" ")

        for phrase in bad_words:
            if process.extractOne(phrase, speech_list)[1] >= 90:
                speech_text = "!random"
                break

        for word in speech_list:
            if word.lower() in bad_words:
                speech_text = "!random"
                break

        if kwargs.get("speech_text", None):
            kwargs["speech_text"] = speech_text
        else:
            args[1] = speech_text

        return tuple(args)[1:], kwargs

    def post_to_slack(self, speech_text, record_utterance=False, **kwargs):
        #Skip plugins
        if speech_text[0] == "!":
            return

        if config.get("slack_url", None) and record_utterance:
            requests.post(config["slack_url"], data=speech_text)

    def record_utterance(self, speech_text, record_utterance=False, event_result=None):
        if not record_utterance or speech_text[0] == "!":
            return

        speech_text = speech_text.lower()

        sha = sha256()
        sha.update(speech_text)
        sha_hash = sha.hexdigest()

        matched_snippet = self.db.execute("SELECT votes FROM snippets where sha256=?", [sha_hash]).fetchone()

        if matched_snippet:
            votes = matched_snippet["votes"] + 1
            self.db.execute("UPDATE snippets set votes=? where sha256=?", [votes, sha_hash])
        else:
            self.db.execute("INSERT into snippets (sha256, speech_text, votes) VALUES (?, ?, 0) ", [sha_hash, speech_text])

    def record_sound_event(self, sound_name, event_result=None):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [sound_name]).fetchone()

        if matched_sound:
            votes = matched_sound["votes"] + 1

            self.db.execute("UPDATE sounds set votes=? where name=?", [votes, sound_name])

    def queue_speech_for_tweet(self, speech_text, record_utterance=False, event_result=None):

        print record_utterance
        if not record_utterance:
            return

        if speech_text[0] == "!":
            return

        speech_text = speech_text[:139].replace("@","~")

        md5_hash = md5()
        md5_hash.update(speech_text.lower())
        md5_hash = md5_hash.hexdigest()

        try:
            check_existence = self.db.execute("SELECT * FROM publish_queue where md5_hash=?", [md5_hash]).next()
        except StopIteration:
            check_existence = False

        if not check_existence:

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [speech_text, md5_hash])

    def queue_sound_for_tweet(self, name, event_result):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

        if matched_sound:
            text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
            md5_hash = md5()
            md5_hash.update(text)
            md5_hash = md5_hash.hexdigest()

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [text, md5_hash])

<|endoftext|>


<commit_before>from __future__ import print_function
import copy
import distutils
import os
from os import path
import pkg_resources
import shutil
import subprocess
import sys
import tempfile

import setuptools
from setuptools.command import build_ext

dummy_extension = setuptools.Extension('chainer', ['chainer.c'])

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_runtime.h',
            'curand.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
        ],
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
    }
]

def get_compiler_setting():
    nvcc_path = search_on_path(('nvcc', 'nvcc.exe'))
    cuda_path_default = None
    if nvcc_path is None:
        print('**************************************************************')
        print('*** WARNING: nvcc not in path.')
        print('*** WARNING: Please set path to nvcc.')
        print('**************************************************************')
    else:
        cuda_path_default = path.normpath(
            path.join(path.dirname(nvcc_path), '..'))

    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
    if len(cuda_path) > 0 and cuda_path != cuda_path_default:
        print('**************************************************************')
        print('*** WARNING: nvcc path != CUDA_PATH')
        print('*** WARNING: nvcc path: %s', cuda_path_default)
        print('*** WARNING: CUDA_PATH: %s', cuda_path)
        print('**************************************************************')

    if not path.exists(cuda_path):
        cuda_path = cuda_path_default

    include_dirs = []
    library_dirs = []
    define_macros = []

    if cuda_path:
        include_dirs.append(path.join(cuda_path, 'include'))
        if sys.platform == 'win32':
            library_dirs.append(path.join(cuda_path, 'bin'))
            library_dirs.append(path.join(cuda_path, 'lib', 'x64'))
        elif sys.platform == 'darwin':
            library_dirs.append(path.join(cuda_path, 'lib'))
        else:
            library_dirs.append(path.join(cuda_path, 'lib64'))
    if sys.platform == 'win32':
        include_dirs.append(localpath('windows'))
    elif sys.platform == 'darwin':
        library_dirs.append('/usr/local/cuda/lib')

    return {
        'include_dirs': include_dirs,
        'library_dirs': library_dirs,
        'define_macros': define_macros,
        'language': 'c++',
    }

def localpath(*args):
    return path.abspath(path.join(path.dirname(__file__), *args))

def get_path(key):
    return os.environ.get(key, '').split(os.pathsep)

def search_on_path(filenames):
    for p in get_path('PATH'):
        for filename in filenames:
            full = path.join(p, filename)
            if path.exists(full):
                return path.abspath(full)

def check_include(dirs, file_path):
    return any(path.exists(path.join(dir, file_path)) for dir in dirs)

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=[], libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        source = '''
        int main(int argc, char* argv[]) {
          return 0;
        }
        '''
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            for header in includes:
                f.write('#include <%s>\n' % header)
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return False

        try:
            compiler.link_shared_lib(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs)
        except (distutils.errors.LinkError, TypeError):
            return False

        return True

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def make_extensions(options, compiler):

    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = get_compiler_setting()

    try:
        import numpy
        numpy_include = numpy.get_include()
    except AttributeError:
        # if numpy is not installed get the headers from the .egg directory
        import numpy.core
        numpy_include = path.join(
            path.dirname(numpy.core.__file__), 'include')
    include_dirs = settings['include_dirs']
    include_dirs.append(numpy_include)

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                print('**************************************************')
                print('*** Include files not found: %s' % module['include'])
                print('*** Skip installing %s support' % module['name'])
                print('*** Check your CPATH environment variable')
                print('**************************************************')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                print('**************************************************')
                print('*** Cannot link libraries: %s' % module['libraries'])
                print('*** Skip installing %s support' % module['name'])
                print('*** Check your LIBRARY_PATH environment variable')
                print('**************************************************')
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']
        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + '.pyx'], **s)
            for f in module['file']])
    return ret

_arg_options = {}

def parse_args():
    global _arg_options
    _arg_options['profile'] = '--cupy-profile' in sys.argv
    if _arg_options['profile']:
        sys.argv.remove('--cupy-profile')

    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    _arg_options['linetrace'] = cupy_coverage
    _arg_options['annotate'] = cupy_coverage

    _arg_options['no_cuda'] = '--cupy-no-cuda' in sys.argv
    if _arg_options['no_cuda']:
        sys.argv.remove('--cupy-no-cuda')
    if check_readthedocs_environment():
        _arg_options['no_cuda'] = True

def cythonize(extensions, force=False, annotate=False, compiler_directives={}):
    cython_pkg = pkg_resources.get_distribution('cython')
    cython_path = path.join(cython_pkg.location, 'cython.py')
    print("cython path:%s" % cython_pkg.location)
    cython_cmdbase = [sys.executable, cython_path]
    subprocess.check_call(cython_cmdbase + ['--version'])

    cython_cmdbase.extend(['--fast-fail', '--verbose', '--cplus'])
    ret = []
    for ext in extensions:
        cmd = list(cython_cmdbase)
        for i in compiler_directives.items():
            cmd.append('--directive')
            cmd.append('%s=%s' % i)
        cpp_files = [path.splitext(f)[0] + ".cpp" for f in ext.sources]
        cmd += ext.sources
        subprocess.check_call(cmd)
        ext = copy.copy(ext)
        ext.sources = cpp_files
        ret.append(ext)
    return ret

class chainer_build_ext(build_ext.build_ext):

    """`build_ext` command for cython files."""

    def finalize_options(self):
        ext_modules = self.distribution.ext_modules
        if dummy_extension in ext_modules:
            print('Executing cythonize()')
            print('Options:', _arg_options)

            directive_keys = ('linetrace', 'profile')
            directives = {key: _arg_options[key] for key in directive_keys}

            cythonize_option_keys = ('annotate',)
            cythonize_options = {
                key: _arg_options[key] for key in cythonize_option_keys}

            compiler = distutils.ccompiler.new_compiler(self.compiler)
            distutils.sysconfig.customize_compiler(compiler)

            extensions = make_extensions(_arg_options, compiler)
            extensions = cythonize(
                extensions,
                force=True,
                compiler_directives=directives,
                **cythonize_options)

            # Modify ext_modules for cython
            ext_modules.remove(dummy_extension)
            ext_modules.extend(extensions)

        build_ext.build_ext.finalize_options(self)
<commit_msg>Remove unused path<commit_after>from __future__ import print_function
import copy
import distutils
import os
from os import path
import pkg_resources
import shutil
import subprocess
import sys
import tempfile

import setuptools
from setuptools.command import build_ext

dummy_extension = setuptools.Extension('chainer', ['chainer.c'])

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_runtime.h',
            'curand.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
        ],
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
    }
]

def get_compiler_setting():
    nvcc_path = search_on_path(('nvcc', 'nvcc.exe'))
    cuda_path_default = None
    if nvcc_path is None:
        print('**************************************************************')
        print('*** WARNING: nvcc not in path.')
        print('*** WARNING: Please set path to nvcc.')
        print('**************************************************************')
    else:
        cuda_path_default = path.normpath(
            path.join(path.dirname(nvcc_path), '..'))

    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
    if len(cuda_path) > 0 and cuda_path != cuda_path_default:
        print('**************************************************************')
        print('*** WARNING: nvcc path != CUDA_PATH')
        print('*** WARNING: nvcc path: %s', cuda_path_default)
        print('*** WARNING: CUDA_PATH: %s', cuda_path)
        print('**************************************************************')

    if not path.exists(cuda_path):
        cuda_path = cuda_path_default

    include_dirs = []
    library_dirs = []
    define_macros = []

    if cuda_path:
        include_dirs.append(path.join(cuda_path, 'include'))
        if sys.platform == 'win32':
            library_dirs.append(path.join(cuda_path, 'bin'))
            library_dirs.append(path.join(cuda_path, 'lib', 'x64'))
        elif sys.platform == 'darwin':
            library_dirs.append(path.join(cuda_path, 'lib'))
        else:
            library_dirs.append(path.join(cuda_path, 'lib64'))
    if sys.platform == 'darwin':
        library_dirs.append('/usr/local/cuda/lib')

    return {
        'include_dirs': include_dirs,
        'library_dirs': library_dirs,
        'define_macros': define_macros,
        'language': 'c++',
    }

def localpath(*args):
    return path.abspath(path.join(path.dirname(__file__), *args))

def get_path(key):
    return os.environ.get(key, '').split(os.pathsep)

def search_on_path(filenames):
    for p in get_path('PATH'):
        for filename in filenames:
            full = path.join(p, filename)
            if path.exists(full):
                return path.abspath(full)

def check_include(dirs, file_path):
    return any(path.exists(path.join(dir, file_path)) for dir in dirs)

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=[], libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        source = '''
        int main(int argc, char* argv[]) {
          return 0;
        }
        '''
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            for header in includes:
                f.write('#include <%s>\n' % header)
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return False

        try:
            compiler.link_shared_lib(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs)
        except (distutils.errors.LinkError, TypeError):
            return False

        return True

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def make_extensions(options, compiler):

    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = get_compiler_setting()

    try:
        import numpy
        numpy_include = numpy.get_include()
    except AttributeError:
        # if numpy is not installed get the headers from the .egg directory
        import numpy.core
        numpy_include = path.join(
            path.dirname(numpy.core.__file__), 'include')
    include_dirs = settings['include_dirs']
    include_dirs.append(numpy_include)

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                print('**************************************************')
                print('*** Include files not found: %s' % module['include'])
                print('*** Skip installing %s support' % module['name'])
                print('*** Check your CPATH environment variable')
                print('**************************************************')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                print('**************************************************')
                print('*** Cannot link libraries: %s' % module['libraries'])
                print('*** Skip installing %s support' % module['name'])
                print('*** Check your LIBRARY_PATH environment variable')
                print('**************************************************')
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']
        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + '.pyx'], **s)
            for f in module['file']])
    return ret

_arg_options = {}

def parse_args():
    global _arg_options
    _arg_options['profile'] = '--cupy-profile' in sys.argv
    if _arg_options['profile']:
        sys.argv.remove('--cupy-profile')

    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    _arg_options['linetrace'] = cupy_coverage
    _arg_options['annotate'] = cupy_coverage

    _arg_options['no_cuda'] = '--cupy-no-cuda' in sys.argv
    if _arg_options['no_cuda']:
        sys.argv.remove('--cupy-no-cuda')
    if check_readthedocs_environment():
        _arg_options['no_cuda'] = True

def cythonize(extensions, force=False, annotate=False, compiler_directives={}):
    cython_pkg = pkg_resources.get_distribution('cython')
    cython_path = path.join(cython_pkg.location, 'cython.py')
    print("cython path:%s" % cython_pkg.location)
    cython_cmdbase = [sys.executable, cython_path]
    subprocess.check_call(cython_cmdbase + ['--version'])

    cython_cmdbase.extend(['--fast-fail', '--verbose', '--cplus'])
    ret = []
    for ext in extensions:
        cmd = list(cython_cmdbase)
        for i in compiler_directives.items():
            cmd.append('--directive')
            cmd.append('%s=%s' % i)
        cpp_files = [path.splitext(f)[0] + ".cpp" for f in ext.sources]
        cmd += ext.sources
        subprocess.check_call(cmd)
        ext = copy.copy(ext)
        ext.sources = cpp_files
        ret.append(ext)
    return ret

class chainer_build_ext(build_ext.build_ext):

    """`build_ext` command for cython files."""

    def finalize_options(self):
        ext_modules = self.distribution.ext_modules
        if dummy_extension in ext_modules:
            print('Executing cythonize()')
            print('Options:', _arg_options)

            directive_keys = ('linetrace', 'profile')
            directives = {key: _arg_options[key] for key in directive_keys}

            cythonize_option_keys = ('annotate',)
            cythonize_options = {
                key: _arg_options[key] for key in cythonize_option_keys}

            compiler = distutils.ccompiler.new_compiler(self.compiler)
            distutils.sysconfig.customize_compiler(compiler)

            extensions = make_extensions(_arg_options, compiler)
            extensions = cythonize(
                extensions,
                force=True,
                compiler_directives=directives,
                **cythonize_options)

            # Modify ext_modules for cython
            ext_modules.remove(dummy_extension)
            ext_modules.extend(extensions)

        build_ext.build_ext.finalize_options(self)
<|endoftext|>


<commit_before>from __future__ import print_function
import copy
import distutils
import os
from os import path
import shutil
import subprocess
import sys
import tempfile

import pkg_resources
import setuptools
from setuptools.command import build_ext

dummy_extension = setuptools.Extension('chainer', ['chainer.c'])

cython_version = '0.23.0'
minimum_cuda_version = 6050
minimum_cudnn_version = 2000

def print_warning(*lines):
    print('**************************************************')
    for line in lines:
        print('*** WARNING: %s' % line)
    print('**************************************************')

def check_cuda_version(compiler, settings):
    out = build_and_run(compiler, '''
    #include <cuda.h>
    #include <stdio.h>
    int main(int argc, char* argv[]) {
      printf("%d", CUDA_VERSION);
      return 0;
    }
    ''', include_dirs=settings['include_dirs'])

    if out is None:
        print_warning('Cannot check CUDA version')
        return False

    cuda_version = int(out)

    if cuda_version < minimum_cuda_version:
        print_warning(
            'CUDA version is too old: %d' % cuda_version,
            'CUDA v6.5 or newer is required')
        return False

    return True

def check_cudnn_version(compiler, settings):
    out = build_and_run(compiler, '''
    #include <cudnn.h>
    #include <stdio.h>
    int main(int argc, char* argv[]) {
      printf("%d", CUDNN_VERSION);
      return 0;
    }
    ''', include_dirs=settings['include_dirs'])

    if out is None:
        print_warning('Cannot check cuDNN version')
        return False

    cudnn_version = int(out)

    if cudnn_version < minimum_cudnn_version:
        print_warning(
            'cuDNN version is too old: %d' % cudnn_version,
            'cuDNN v2 or newer is required')
        return False

    return True

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.core.internal',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_runtime.h',
            'curand.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
        ],
        'check_method': check_cuda_version,
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
        'check_method': check_cudnn_version,
    }
]

def get_compiler_setting():
    nvcc_path = search_on_path(('nvcc', 'nvcc.exe'))
    cuda_path_default = None
    if nvcc_path is None:
        print_warning('nvcc not in path.',
                      'Please set path to nvcc.')
    else:
        cuda_path_default = path.normpath(
            path.join(path.dirname(nvcc_path), '..'))

    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
    if len(cuda_path) > 0 and cuda_path != cuda_path_default:
        print_warning(
            'nvcc path != CUDA_PATH',
            'nvcc path: %s' % cuda_path_default,
            'CUDA_PATH: %s' % cuda_path)

    if not path.exists(cuda_path):
        cuda_path = cuda_path_default

    if not cuda_path and path.exists('/usr/local/cuda'):
        cuda_path = '/usr/local/cuda'

    include_dirs = []
    library_dirs = []
    define_macros = []

    if cuda_path:
        include_dirs.append(path.join(cuda_path, 'include'))
        if sys.platform == 'win32':
            library_dirs.append(path.join(cuda_path, 'bin'))
            library_dirs.append(path.join(cuda_path, 'lib', 'x64'))
        else:
            library_dirs.append(path.join(cuda_path, 'lib64'))
            library_dirs.append(path.join(cuda_path, 'lib'))
    if sys.platform == 'darwin':
        library_dirs.append('/usr/local/cuda/lib')

    return {
        'include_dirs': include_dirs,
        'library_dirs': library_dirs,
        'define_macros': define_macros,
        'language': 'c++',
    }

def localpath(*args):
    return path.abspath(path.join(path.dirname(__file__), *args))

def get_path(key):
    return os.environ.get(key, '').split(os.pathsep)

def search_on_path(filenames):
    for p in get_path('PATH'):
        for filename in filenames:
            full = path.join(p, filename)
            if path.exists(full):
                return path.abspath(full)

def check_include(dirs, file_path):
    return any(path.exists(path.join(dir, file_path)) for dir in dirs)

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=[], libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        source = '''
        int main(int argc, char* argv[]) {
          return 0;
        }
        '''
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            for header in includes:
                f.write('#include <%s>\n' % header)
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return False

        try:
            compiler.link_shared_lib(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs)
        except (distutils.errors.LinkError, TypeError):
            return False

        return True

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def build_and_run(compiler, source, libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return None

        try:
            compiler.link_executable(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs)
        except (distutils.errors.LinkError, TypeError):
            return None

        try:
            out = subprocess.check_output(os.path.join(temp_dir, 'a'))
            return out

        except Exception:
            return None

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def get_numpy_include_path():
    import six
    if hasattr(six.moves.builtins, '__NUMPY_SETUP__'):
        del six.moves.builtins.__NUMPY_SETUP__
    import numpy
    six.moves.reload_module(numpy)
    try:
        numpy_include = numpy.get_include()
    except AttributeError:
        numpy_include = numpy.get_numpy_include()
    return numpy_include

def make_extensions(options, compiler):

    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = get_compiler_setting()

    include_dirs = settings['include_dirs']
    include_dirs.append(get_numpy_include_path())

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                print_warning(
                    'Include files not found: %s' % module['include'],
                    'Skip installing %s support' % module['name'],
                    'Check your CPATH environment variable')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                print_warning(
                    'Cannot link libraries: %s' % module['libraries'],
                    'Skip installing %s support' % module['name'],
                    'Check your LIBRARY_PATH environment variable')
                continue

            if 'check_method' in module and \
               not module['check_method'](compiler, settings):
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']
        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + '.pyx'], **s)
            for f in module['file']])
    return ret

_arg_options = {}

def parse_args():
    global _arg_options
    _arg_options['profile'] = '--cupy-profile' in sys.argv
    if _arg_options['profile']:
        sys.argv.remove('--cupy-profile')

    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    _arg_options['linetrace'] = cupy_coverage
    _arg_options['annotate'] = cupy_coverage

    _arg_options['no_cuda'] = '--cupy-no-cuda' in sys.argv
    if _arg_options['no_cuda']:
        sys.argv.remove('--cupy-no-cuda')
    if check_readthedocs_environment():
        _arg_options['no_cuda'] = True

def get_cython_pkg():
    try:
        return pkg_resources.get_distribution('cython')
    except pkg_resources.DistributionNotFound:
        return None

def run_command(cmd):
    try:
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        msg = '''Command %r failed:

  command: %s
  return code: %d
  output:

%s''' % (cmd[0], ' '.join(e.cmd), e.returncode, e.output)
        raise distutils.errors.DistutilsExecError(msg)

def cythonize(extensions, force=False, annotate=False, compiler_directives={}):
    cython_location = get_cython_pkg().location
    cython_path = path.join(cython_location, 'cython.py')
    print("cython path:%s" % cython_location)
    cython_cmdbase = [sys.executable, cython_path]
    run_command(cython_cmdbase + ['--version'])

    cython_cmdbase.extend(['--fast-fail', '--verbose', '--cplus'])
    for ext in extensions:
        cmd = list(cython_cmdbase)
        for i in compiler_directives.items():
            cmd.append('--directive')
            cmd.append('%s=%s' % i)
        run_command(cmd + ext.sources)

def to_cpp_extensions(extensions):
    ret = []
    for x in extensions:
        ext = copy.copy(x)
        ext.sources = [path.splitext(f)[0] + ".cpp" for f in x.sources]
        ret.append(ext)
    return ret

def check_extensions(extensions):
    for x in extensions:
        for f in x.sources:
            if not path.isfile(f):
                msg = ('Missing file: %s\n' % f +
                       'Please install Cython.\n' +
                       'See http://docs.chainer.org/en/stable/install.html')
                raise RuntimeError(msg)

class chainer_build_ext(build_ext.build_ext):

    """`build_ext` command for cython files."""

    def finalize_options(self):
        ext_modules = self.distribution.ext_modules
        if dummy_extension in ext_modules:
            print('Executing cythonize')
            print('Options:', _arg_options)

            directive_keys = ('linetrace', 'profile')
            directives = {key: _arg_options[key] for key in directive_keys}

            cythonize_option_keys = ('annotate',)
            cythonize_options = {
                key: _arg_options[key] for key in cythonize_option_keys}

            compiler = distutils.ccompiler.new_compiler(self.compiler)
            distutils.sysconfig.customize_compiler(compiler)

            extensions = make_extensions(_arg_options, compiler)

            cython = get_cython_pkg()
            req_version = pkg_resources.parse_version(cython_version)
            if cython is not None and cython.parsed_version > req_version:
                cythonize(extensions, force=True,
                          compiler_directives=directives, **cythonize_options)

            extensions = to_cpp_extensions(extensions)
            check_extensions(extensions)

            # Modify ext_modules for cython
            ext_modules.remove(dummy_extension)
            ext_modules.extend(extensions)

        build_ext.build_ext.finalize_options(self)
<commit_msg>Needed to link with MSVC<commit_after>from __future__ import print_function
import copy
import distutils
import os
from os import path
import shutil
import subprocess
import sys
import tempfile

import pkg_resources
import setuptools
from setuptools.command import build_ext

dummy_extension = setuptools.Extension('chainer', ['chainer.c'])

cython_version = '0.23.0'
minimum_cuda_version = 6050
minimum_cudnn_version = 2000

def print_warning(*lines):
    print('**************************************************')
    for line in lines:
        print('*** WARNING: %s' % line)
    print('**************************************************')

def check_cuda_version(compiler, settings):
    out = build_and_run(compiler, '''
    #include <cuda.h>
    #include <stdio.h>
    int main(int argc, char* argv[]) {
      printf("%d", CUDA_VERSION);
      return 0;
    }
    ''', include_dirs=settings['include_dirs'])

    if out is None:
        print_warning('Cannot check CUDA version')
        return False

    cuda_version = int(out)

    if cuda_version < minimum_cuda_version:
        print_warning(
            'CUDA version is too old: %d' % cuda_version,
            'CUDA v6.5 or newer is required')
        return False

    return True

def check_cudnn_version(compiler, settings):
    out = build_and_run(compiler, '''
    #include <cudnn.h>
    #include <stdio.h>
    int main(int argc, char* argv[]) {
      printf("%d", CUDNN_VERSION);
      return 0;
    }
    ''', include_dirs=settings['include_dirs'])

    if out is None:
        print_warning('Cannot check cuDNN version')
        return False

    cudnn_version = int(out)

    if cudnn_version < minimum_cudnn_version:
        print_warning(
            'cuDNN version is too old: %d' % cudnn_version,
            'cuDNN v2 or newer is required')
        return False

    return True

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.core.internal',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_runtime.h',
            'curand.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
        ],
        'check_method': check_cuda_version,
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
        'check_method': check_cudnn_version,
    }
]

def get_compiler_setting():
    nvcc_path = search_on_path(('nvcc', 'nvcc.exe'))
    cuda_path_default = None
    if nvcc_path is None:
        print_warning('nvcc not in path.',
                      'Please set path to nvcc.')
    else:
        cuda_path_default = path.normpath(
            path.join(path.dirname(nvcc_path), '..'))

    cuda_path = os.environ.get('CUDA_PATH', '')  # Nvidia default on Windows
    if len(cuda_path) > 0 and cuda_path != cuda_path_default:
        print_warning(
            'nvcc path != CUDA_PATH',
            'nvcc path: %s' % cuda_path_default,
            'CUDA_PATH: %s' % cuda_path)

    if not path.exists(cuda_path):
        cuda_path = cuda_path_default

    if not cuda_path and path.exists('/usr/local/cuda'):
        cuda_path = '/usr/local/cuda'

    include_dirs = []
    library_dirs = []
    define_macros = []

    if cuda_path:
        include_dirs.append(path.join(cuda_path, 'include'))
        if sys.platform == 'win32':
            library_dirs.append(path.join(cuda_path, 'bin'))
            library_dirs.append(path.join(cuda_path, 'lib', 'x64'))
        else:
            library_dirs.append(path.join(cuda_path, 'lib64'))
            library_dirs.append(path.join(cuda_path, 'lib'))
    if sys.platform == 'darwin':
        library_dirs.append('/usr/local/cuda/lib')

    return {
        'include_dirs': include_dirs,
        'library_dirs': library_dirs,
        'define_macros': define_macros,
        'language': 'c++',
    }

def localpath(*args):
    return path.abspath(path.join(path.dirname(__file__), *args))

def get_path(key):
    return os.environ.get(key, '').split(os.pathsep)

def search_on_path(filenames):
    for p in get_path('PATH'):
        for filename in filenames:
            full = path.join(p, filename)
            if path.exists(full):
                return path.abspath(full)

def check_include(dirs, file_path):
    return any(path.exists(path.join(dir, file_path)) for dir in dirs)

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=[], libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        source = '''
        int main(int argc, char* argv[]) {
          return 0;
        }
        '''
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            for header in includes:
                f.write('#include <%s>\n' % header)
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return False

        try:
            compiler.link_shared_lib(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs)
        except (distutils.errors.LinkError, TypeError):
            return False

        return True

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def build_and_run(compiler, source, libraries=[],
                  include_dirs=[], library_dirs=[]):
    temp_dir = tempfile.mkdtemp()

    try:
        fname = os.path.join(temp_dir, 'a.cpp')
        with open(fname, 'w') as f:
            f.write(source)

        try:
            objects = compiler.compile([fname], output_dir=temp_dir,
                                       include_dirs=include_dirs)
        except distutils.errors.CompileError:
            return None

        try:
            postargs = ['/MANIFEST'] if sys.platform == 'win32' else []
            compiler.link_executable(objects,
                                     os.path.join(temp_dir, 'a'),
                                     libraries=libraries,
                                     library_dirs=library_dirs,
                                     extra_postargs=postargs)
        except (distutils.errors.LinkError, TypeError):
            return None

        try:
            out = subprocess.check_output(os.path.join(temp_dir, 'a'))
            return out

        except Exception:
            return None

    finally:
        shutil.rmtree(temp_dir, ignore_errors=True)

def get_numpy_include_path():
    import six
    if hasattr(six.moves.builtins, '__NUMPY_SETUP__'):
        del six.moves.builtins.__NUMPY_SETUP__
    import numpy
    six.moves.reload_module(numpy)
    try:
        numpy_include = numpy.get_include()
    except AttributeError:
        numpy_include = numpy.get_numpy_include()
    return numpy_include

def make_extensions(options, compiler):

    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = get_compiler_setting()

    include_dirs = settings['include_dirs']
    include_dirs.append(get_numpy_include_path())

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                print_warning(
                    'Include files not found: %s' % module['include'],
                    'Skip installing %s support' % module['name'],
                    'Check your CPATH environment variable')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                print_warning(
                    'Cannot link libraries: %s' % module['libraries'],
                    'Skip installing %s support' % module['name'],
                    'Check your LIBRARY_PATH environment variable')
                continue

            if 'check_method' in module and \
               not module['check_method'](compiler, settings):
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']
        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + '.pyx'], **s)
            for f in module['file']])
    return ret

_arg_options = {}

def parse_args():
    global _arg_options
    _arg_options['profile'] = '--cupy-profile' in sys.argv
    if _arg_options['profile']:
        sys.argv.remove('--cupy-profile')

    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    _arg_options['linetrace'] = cupy_coverage
    _arg_options['annotate'] = cupy_coverage

    _arg_options['no_cuda'] = '--cupy-no-cuda' in sys.argv
    if _arg_options['no_cuda']:
        sys.argv.remove('--cupy-no-cuda')
    if check_readthedocs_environment():
        _arg_options['no_cuda'] = True

def get_cython_pkg():
    try:
        return pkg_resources.get_distribution('cython')
    except pkg_resources.DistributionNotFound:
        return None

def run_command(cmd):
    try:
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        msg = '''Command %r failed:

  command: %s
  return code: %d
  output:

%s''' % (cmd[0], ' '.join(e.cmd), e.returncode, e.output)
        raise distutils.errors.DistutilsExecError(msg)

def cythonize(extensions, force=False, annotate=False, compiler_directives={}):
    cython_location = get_cython_pkg().location
    cython_path = path.join(cython_location, 'cython.py')
    print("cython path:%s" % cython_location)
    cython_cmdbase = [sys.executable, cython_path]
    run_command(cython_cmdbase + ['--version'])

    cython_cmdbase.extend(['--fast-fail', '--verbose', '--cplus'])
    for ext in extensions:
        cmd = list(cython_cmdbase)
        for i in compiler_directives.items():
            cmd.append('--directive')
            cmd.append('%s=%s' % i)
        run_command(cmd + ext.sources)

def to_cpp_extensions(extensions):
    ret = []
    for x in extensions:
        ext = copy.copy(x)
        ext.sources = [path.splitext(f)[0] + ".cpp" for f in x.sources]
        ret.append(ext)
    return ret

def check_extensions(extensions):
    for x in extensions:
        for f in x.sources:
            if not path.isfile(f):
                msg = ('Missing file: %s\n' % f +
                       'Please install Cython.\n' +
                       'See http://docs.chainer.org/en/stable/install.html')
                raise RuntimeError(msg)

class chainer_build_ext(build_ext.build_ext):

    """`build_ext` command for cython files."""

    def finalize_options(self):
        ext_modules = self.distribution.ext_modules
        if dummy_extension in ext_modules:
            print('Executing cythonize')
            print('Options:', _arg_options)

            directive_keys = ('linetrace', 'profile')
            directives = {key: _arg_options[key] for key in directive_keys}

            cythonize_option_keys = ('annotate',)
            cythonize_options = {
                key: _arg_options[key] for key in cythonize_option_keys}

            compiler = distutils.ccompiler.new_compiler(self.compiler)
            distutils.sysconfig.customize_compiler(compiler)

            extensions = make_extensions(_arg_options, compiler)

            cython = get_cython_pkg()
            req_version = pkg_resources.parse_version(cython_version)
            if cython is not None and cython.parsed_version > req_version:
                cythonize(extensions, force=True,
                          compiler_directives=directives, **cythonize_options)

            extensions = to_cpp_extensions(extensions)
            check_extensions(extensions)

            # Modify ext_modules for cython
            ext_modules.remove(dummy_extension)
            ext_modules.extend(extensions)

        build_ext.build_ext.finalize_options(self)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2004-2013 QUIVAL, S.A. All Rights Reserved
#    $Pedro Gómez Campos$ <pegomez@elnogal.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, orm
from openerp.tools.translate import _

class account_invoice(orm.Model):
    _inherit = "account.invoice"

    # POST-MIGRATION: nUEVA FUNCIONALIDAD ABAJO, ARRASTRA EL MANY2ONE EN LUGAR DEL BOOLEAN
    # def _received_check(self, cr, uid, ids, name, args, context=None):
    #     res = {}
    #     for inv in self.browse(cr, uid, ids, context=context):
    #         res[inv.id] = False
    #         if inv.move_id and inv.move_id.line_id:
    #             for line in inv.move_id.line_id:
    #                 if line.received_check:
    #                     res[inv.id] = True
    #     return res

    def _received_check(self, cr, uid, ids, name, args, context=None):
        res = {}
        for inv in self.browse(cr, uid, ids, context=context):
            res[inv.id] = False
            for payment in inv.payment_ids:
                # if line.received_check:  comentado POST_MIGRATION, no existe campo, se metía en account payment_extension
                if payment.check_deposit_id:
                    res[inv.id] = payment.check_deposit_id.id
        return res

    _columns = {
        # 'received_check': fields.function(_received_check, method=True, store=False, type='boolean', string='Received check', help="To write down that a check in paper support has been received, for example."),
        'received_check': fields.function(_received_check, method=True, store=False, type='many2one', relation='account.check.deposit',
                                          string='Received check', help="To write down that a check in paper support has been received, for example.")
    }

    def _refund_cleanup_lines(self, cr, uid, lines):
        """ugly function to map all fields of account.invoice.line when creates refund invoice"""
        res = super(account_invoice, self)._refund_cleanup_lines(cr, uid, lines)

        for line in res:
            if 'tax_id' in line[2]:
                line[2]['tax_id'] = line[2]['tax_id'] and line[2]['tax_id'][0] or False

        return res

class account_invoice_line(orm.Model):
    _inherit = "account.invoice.line"

    def uos_id_change(self, cr, uid, ids, product, uom, qty=0, name='', type='out_invoice', partner_id=False, fposition_id=False, price_unit=False, address_invoice_id=False, currency_id=False, context=None, company_id=None):
        """
        Modificamos para no permitir poner cualquier tipo de unidad en la linea de factura.
        Ahora solo aceptará alguna que pertenezca a la misma categoria que la unidad de medida por defecto del producto.
        """
        if context is None:
            context = {}

        update_res = False

        if product and uom:
            prod = self.pool.get('product.product').browse(cr, uid, product, context=context)
            prod_uom = self.pool.get('product.uom').browse(cr, uid, uom, context=context)
            if prod.uom_id.category_id.id != prod_uom.category_id.id:
                update_res = True
                uom = (prod.uos_id and prod.uos_id.id) or (prod.uom_id and prod.uom_id.id) or False

        res = super(account_invoice_line, self).uos_id_change(cr, uid, ids, product, uom, qty, name, type, partner_id, fposition_id, price_unit, address_invoice_id, currency_id, context, company_id)

        if update_res:
            res['value']['uos_id'] = uom or False

        return res<commit_msg>[FIX]. eln_account. Generar rectificativas<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2004-2013 QUIVAL, S.A. All Rights Reserved
#    $Pedro Gómez Campos$ <pegomez@elnogal.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, orm
from openerp.tools.translate import _

class account_invoice(orm.Model):
    _inherit = "account.invoice"

    # POST-MIGRATION: nUEVA FUNCIONALIDAD ABAJO, ARRASTRA EL MANY2ONE EN LUGAR DEL BOOLEAN
    # def _received_check(self, cr, uid, ids, name, args, context=None):
    #     res = {}
    #     for inv in self.browse(cr, uid, ids, context=context):
    #         res[inv.id] = False
    #         if inv.move_id and inv.move_id.line_id:
    #             for line in inv.move_id.line_id:
    #                 if line.received_check:
    #                     res[inv.id] = True
    #     return res

    def _received_check(self, cr, uid, ids, name, args, context=None):
        res = {}
        for inv in self.browse(cr, uid, ids, context=context):
            res[inv.id] = False
            for payment in inv.payment_ids:
                # if line.received_check:  comentado POST_MIGRATION, no existe campo, se metía en account payment_extension
                if payment.check_deposit_id:
                    res[inv.id] = payment.check_deposit_id.id
        return res

    _columns = {
        # 'received_check': fields.function(_received_check, method=True, store=False, type='boolean', string='Received check', help="To write down that a check in paper support has been received, for example."),
        'received_check': fields.function(_received_check, method=True, store=False, type='many2one', relation='account.check.deposit',
                                          string='Received check', help="To write down that a check in paper support has been received, for example.")
    }

    def _refund_cleanup_lines(self, lines):
        """ugly function to map all fields of account.invoice.line when creates refund invoice"""
        res = super(account_invoice, self)._refund_cleanup_lines(lines)

        for line in res:
            if 'tax_id' in line[2]:
                line[2]['tax_id'] = line[2]['tax_id'] and line[2]['tax_id'][0] or False

        return res

class account_invoice_line(orm.Model):
    _inherit = "account.invoice.line"

    def uos_id_change(self, cr, uid, ids, product, uom, qty=0, name='', type='out_invoice', partner_id=False, fposition_id=False, price_unit=False, address_invoice_id=False, currency_id=False, context=None, company_id=None):
        """
        Modificamos para no permitir poner cualquier tipo de unidad en la linea de factura.
        Ahora solo aceptará alguna que pertenezca a la misma categoria que la unidad de medida por defecto del producto.
        """
        if context is None:
            context = {}

        update_res = False

        if product and uom:
            prod = self.pool.get('product.product').browse(cr, uid, product, context=context)
            prod_uom = self.pool.get('product.uom').browse(cr, uid, uom, context=context)
            if prod.uom_id.category_id.id != prod_uom.category_id.id:
                update_res = True
                uom = (prod.uos_id and prod.uos_id.id) or (prod.uom_id and prod.uom_id.id) or False

        res = super(account_invoice_line, self).uos_id_change(cr, uid, ids, product, uom, qty, name, type, partner_id, fposition_id, price_unit, address_invoice_id, currency_id, context, company_id)

        if update_res:
            res['value']['uos_id'] = uom or False

        return res<|endoftext|>


<commit_before>import re
from itertools import chain
from operator import attrgetter

from circuits.protocols.irc import joinprefix, Message

from ..events import signon
from ..plugin import BasePlugin
from ..models import User, UserInfo
from ..commands import BaseCommands
from ..replies import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE

VALID_NICK_REGEX = re.compile(r"^[][\`_^{|}A-Za-z][][\`_^{|}A-Za-z0-9-]*$")

class Commands(BaseCommands):

    def quit(self, sock, source, reason=u"Leaving"):
        user = User.objects.filter(sock=sock).first()

        for channel in user.channels:
            channel.users.remove(user)

            if not channel.users:
                channel.delete()

        users = chain(*map(attrgetter("users"), user.channels))

        self.disconnect(user)

        self.notify(users, Message(u"QUIT", reason, prefix=user.prefix), user)

    def nick(self, sock, source, nick):
        user = User.objects.filter(sock=sock).first()

        if not VALID_NICK_REGEX.match(nick):
            return ERR_ERRONEUSNICKNAME(nick)

        if len(nick) > self.parent.nicklen:
            return ERR_ERRONEUSNICKNAME(nick)

        if any(x for x in User.objects.all() if x.nick and x.nick.lower() == nick):
            return ERR_NICKNAMEINUSE(nick)

        prefix = user.prefix or joinprefix(*source)
        user.nick = nick
        user.save()

        if not user.registered and user.userinfo is not None:
            user.registered = True
            user.save()
            return signon(sock, user.source)

        users = chain(*map(attrgetter("users"), user.channels))

        self.notify(users, Message(u"NICK", nick, prefix=prefix))

    def user(self, sock, source, username, hostname, server, realname):
        _user = User.objects.filter(sock=sock).first()

        if _user.userinfo is None:
            userinfo = UserInfo(
                user=username, host=hostname, name=realname, server=server
            )
        else:
            userinfo = _user.userinfo
            userinfo.user = username
            userinfo.name = realname
            userinfo.server = server

        userinfo.save()

        _user.userinfo = userinfo
        _user.save()

        if not _user.registered and _user.nick:
            _user.registered = True
            _user.save()
            return signon(sock, _user.source)

class Core(BasePlugin):

    def init(self, *args, **kwargs):
        super(Core, self).init(*args, **kwargs)

        self.nicklen = 16

        self.features = (
            "NICKLEN={0}".format(self.nicklen),
        )

        Commands(*args, **kwargs).register(self)

    def supports(self):
        return self.features
<commit_msg>Fix nick collision detection proper (case\!)<commit_after>import re
from itertools import chain
from operator import attrgetter

from circuits.protocols.irc import joinprefix, Message

from ..events import signon
from ..plugin import BasePlugin
from ..models import User, UserInfo
from ..commands import BaseCommands
from ..replies import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE

VALID_NICK_REGEX = re.compile(r"^[][\`_^{|}A-Za-z][][\`_^{|}A-Za-z0-9-]*$")

class Commands(BaseCommands):

    def quit(self, sock, source, reason=u"Leaving"):
        user = User.objects.filter(sock=sock).first()

        for channel in user.channels:
            channel.users.remove(user)

            if not channel.users:
                channel.delete()

        users = chain(*map(attrgetter("users"), user.channels))

        self.disconnect(user)

        self.notify(users, Message(u"QUIT", reason, prefix=user.prefix), user)

    def nick(self, sock, source, nick):
        user = User.objects.filter(sock=sock).first()

        if not VALID_NICK_REGEX.match(nick):
            return ERR_ERRONEUSNICKNAME(nick)

        if len(nick) > self.parent.nicklen:
            return ERR_ERRONEUSNICKNAME(nick)

        if any(x for x in User.objects.all() if x.nick and x.nick.lower() == nick.lower()):
            return ERR_NICKNAMEINUSE(nick)

        prefix = user.prefix or joinprefix(*source)
        user.nick = nick
        user.save()

        if not user.registered and user.userinfo is not None:
            user.registered = True
            user.save()
            return signon(sock, user.source)

        users = chain(*map(attrgetter("users"), user.channels))

        self.notify(users, Message(u"NICK", nick, prefix=prefix))

    def user(self, sock, source, username, hostname, server, realname):
        _user = User.objects.filter(sock=sock).first()

        if _user.userinfo is None:
            userinfo = UserInfo(
                user=username, host=hostname, name=realname, server=server
            )
        else:
            userinfo = _user.userinfo
            userinfo.user = username
            userinfo.name = realname
            userinfo.server = server

        userinfo.save()

        _user.userinfo = userinfo
        _user.save()

        if not _user.registered and _user.nick:
            _user.registered = True
            _user.save()
            return signon(sock, _user.source)

class Core(BasePlugin):

    def init(self, *args, **kwargs):
        super(Core, self).init(*args, **kwargs)

        self.nicklen = 16

        self.features = (
            "NICKLEN={0}".format(self.nicklen),
        )

        Commands(*args, **kwargs).register(self)

    def supports(self):
        return self.features
<|endoftext|>


<commit_before>#!/usr/bin/env python
# define some variables
import pmag,sys,pmagplotlib,continents
def main():
    """
    NAME
        vgpmap_magic.py

    DESCRIPTION
        makes a map of vgps and a95/dp,dm for site means in a pmag_results table

    SYNTAX
        vgpmap_magic.py [command line options]

    OPTIONS
        -h prints help and quits
        -eye  ELAT ELON [specify eyeball location], default is 90., 0.
        -f FILE pmag_results format file, [default is pmag_results.txt]
        -res [c,l,i,h] specify resolution (crude, low, intermediate, high]
        -etp plot the etopo20 topographpy data (requires high resolution data set)
        -prj PROJ,  specify one of the following:
             ortho = orthographic
             lcc = lambert conformal
             moll = molweide
             merc = mercator
        -sym SYM SIZE: choose a symbol and size, examples:
            ro 5 : small red circles
            bs 10 : intermediate blue squares
            g^ 20 : large green triangles
        -ell  plot dp/dm or a95 ellipses
        -rev RSYM RSIZE : flip reverse poles to normal antipode
        -S:  plot antipodes of all poles
        -age : plot the ages next to the poles
        -crd [g,t] : choose coordinate system, default is to plot all site VGPs
        -fmt [pdf, png, eps...] specify output format, default is pdf

    DEFAULTS
        FILE: pmag_results.txt
        res:  c
        prj: mercator
        ELAT,ELON = 0,0
        SYM SIZE: ro 8
        RSYM RSIZE: g^ 8

    """
    dir_path='.'
    res,ages='c',0
    proj='npstere'
    results_file='pmag_results.txt'
    ell,flip=0,0
    lat_0,lon_0=90.,0.
    fmt='pdf'
    sym,size='ro',8
    rsym,rsize='g^',8
    anti=0
    fancy=0
    coord=""
    if '-WD' in sys.argv:
        ind = sys.argv.index('-WD')
        dir_path=sys.argv[ind+1]
    if '-h' in sys.argv:
        print main.__doc__
        sys.exit()
    if '-S' in sys.argv:anti=1
    if '-fmt' in sys.argv:
        ind = sys.argv.index('-fmt')
        fmt=sys.argv[ind+1]
    if '-res' in sys.argv:
        ind = sys.argv.index('-res')
        res=sys.argv[ind+1]
    if '-etp' in sys.argv:fancy=1
    if '-prj' in sys.argv:
        ind = sys.argv.index('-prj')
        proj=sys.argv[ind+1]
    if '-rev' in sys.argv:
        flip=1
        ind = sys.argv.index('-rev')
        rsym=(sys.argv[ind+1])
        rsize=int(sys.argv[ind+2])
    if '-sym' in sys.argv:
        ind = sys.argv.index('-sym')
        sym=(sys.argv[ind+1])
        size=int(sys.argv[ind+2])
    if '-eye' in sys.argv:
        ind = sys.argv.index('-eye')
        lat_0=float(sys.argv[ind+1])
        lon_0=float(sys.argv[ind+2])
    if '-ell' in sys.argv: ell=1
    if '-age' in sys.argv: ages=1
    if '-f' in sys.argv:
        ind = sys.argv.index('-f')
        results_file=sys.argv[ind+1]
    if '-crd' in sys.argv:
        ind = sys.argv.index('-crd')
        crd=sys.argv[ind+1]
        if crd=='g':coord='0'
        if crd=='t':coord='100'
    results_file=dir_path+'/'+results_file
    data,file_type=pmag.magic_read(results_file)
    if file_type!='pmag_results':
        print "bad results file"
        sys.exit()
    FIG={'map':1}
    pmagplotlib.plot_init(FIG['map'],6,6)
    # read in er_sites file
    lats,lons,dp,dm,a95=[],[],[],[],[]
    Pars=[]
    dates,rlats,rlons=[],[],[]
    Results=pmag.get_dictitem(data,'data_type','i','T') # get all site level data
    Results=pmag.get_dictitem(Results,'vgp_lat','','F') # get all non-blank latitudes
    Results=pmag.get_dictitem(Results,'vgp_lon','','F') # get all non-blank longitudes
    if coord!="":Results=pmag.get_dictitem(Results,'tilt_correction',coord,'T') # get specified coordinate system
    for rec in Results:
            if 'average_age' in rec.keys() and rec['average_age']!="" and ages==1:
                dates.append(rec['average_age'])
            lat=float(rec['vgp_lat'])
            lon=float(rec['vgp_lon'])
            if flip==0:
                lats.append(lat)
                lons.append(lon)
            elif flip==1:
                if lat<0:
                    rlats.append(-lat)
                    lon=lon+180.
                    if lon>360:lon=lon-360.
                    rlons.append(lon)
                else:
                    lats.append(lat)
                    lons.append(lon)
            elif anti==1:
                lats.append(-lat)
                lon=lon+180.
                if lon>360:lon=lon-360.
                lons.append(lon)
            ppars=[]
            ppars.append(lon)
            ppars.append(lat)
            ell1,ell2="",""
            if 'vgp_dm' in rec.keys() and rec['vgp_dm']!="":ell1=float(rec['vgp_dm'])
            if 'vgp_dp' in rec.keys() and rec['vgp_dp']!="":ell2=float(rec['vgp_dp'])
            if 'vgp_alpha95' in rec.keys() and rec['vgp_alpha95']!="":ell1,ell2=float(rec['vgp_alpha95']),float(rec['vgp_alpha95'])
            if ell1!="" and ell2!="":
                ppars=[]
                ppars.append(lons[-1])
                ppars.append(lats[-1])
                ppars.append(ell1)
                ppars.append(lons[-1])
                isign=abs(lats[-1])/lats[-1]
                ppars.append(lats[-1]-isign*90.)
                ppars.append(ell2)
                ppars.append(lons[-1]+90.)
                ppars.append(0.)
                Pars.append(ppars)
    Opts={'latmin':-90,'latmax':90,'lonmin':0.,'lonmax':360.,'lat_0':lat_0,'lon_0':lon_0,'proj':proj,'sym':'bs','symsize':3,'pltgrid':0,'res':res,'boundinglat':0.}
    Opts['details']={'coasts':1,'rivers':0, 'states':0, 'countries':0,'ocean':1,'fancy':fancy}
    pmagplotlib.plotMAP(FIG['map'],[90.],[0.],Opts) # make the base map with a blue triangle at the pole`
    Opts['pltgrid']=-1
    Opts['sym']=sym
    Opts['symsize']=size
    if len(dates)>0:Opts['names']=dates
    if len(lats)>0:pmagplotlib.plotMAP(FIG['map'],lats,lons,Opts) # add the lats and lons of the poles
    Opts['names']=[]
    if len(rlats)>0:
        Opts['sym']=rsym
        Opts['symsize']=rsize
        pmagplotlib.plotMAP(FIG['map'],rlats,rlons,Opts) # add the lats and lons of the poles
    pmagplotlib.drawFIGS(FIG)
    if ell==1: # add ellipses if desired.
        Opts['details']={'coasts':0,'rivers':0, 'states':0, 'countries':0,'ocean':0}
        Opts['pltgrid']=-1 # turn off meridian replotting
        Opts['symsize']=2
        Opts['sym']='g-'
        for ppars in Pars:
            if ppars[2]!=0:
                PTS=pmagplotlib.plotELL(FIG['map'],ppars,'g.',0,0)
                elats,elons=[],[]
                for pt in PTS:
                    elons.append(pt[0])
                    elats.append(pt[1])
                pmagplotlib.plotMAP(FIG['map'],elats,elons,Opts) # make the base map with a blue triangle at the pole`
                pmagplotlib.drawFIGS(FIG)
    files={}
    for key in FIG.keys():
        files[key]='VGP_map'+'.'+fmt
    if pmagplotlib.isServer:
        black     = '#000000'
        purple    = '#800080'
        titles={}
        titles['eq']='VGP Map'
        FIG = pmagplotlib.addBorders(FIG,titles,black,purple)
        pmagplotlib.saveP(FIG,files)
    else:
        ans=raw_input(" S[a]ve to save plot, Return to quit:  ")
        if ans=="a":
            pmagplotlib.saveP(FIG,files)
        else:
            print "Good bye"
            sys.exit()
main()
<commit_msg>fixed bug in vgpmap_magic.py<commit_after>#!/usr/bin/env python
# define some variables
import pmag,sys,pmagplotlib,continents
def main():
    """
    NAME
        vgpmap_magic.py

    DESCRIPTION
        makes a map of vgps and a95/dp,dm for site means in a pmag_results table

    SYNTAX
        vgpmap_magic.py [command line options]

    OPTIONS
        -h prints help and quits
        -eye  ELAT ELON [specify eyeball location], default is 90., 0.
        -f FILE pmag_results format file, [default is pmag_results.txt]
        -res [c,l,i,h] specify resolution (crude, low, intermediate, high]
        -etp plot the etopo20 topographpy data (requires high resolution data set)
        -prj PROJ,  specify one of the following:
             ortho = orthographic
             lcc = lambert conformal
             moll = molweide
             merc = mercator
        -sym SYM SIZE: choose a symbol and size, examples:
            ro 5 : small red circles
            bs 10 : intermediate blue squares
            g^ 20 : large green triangles
        -ell  plot dp/dm or a95 ellipses
        -rev RSYM RSIZE : flip reverse poles to normal antipode
        -S:  plot antipodes of all poles
        -age : plot the ages next to the poles
        -crd [g,t] : choose coordinate system, default is to plot all site VGPs
        -fmt [pdf, png, eps...] specify output format, default is pdf

    DEFAULTS
        FILE: pmag_results.txt
        res:  c
        prj: mercator
        ELAT,ELON = 0,0
        SYM SIZE: ro 8
        RSYM RSIZE: g^ 8

    """
    dir_path='.'
    res,ages='c',0
    proj='npstere'
    results_file='pmag_results.txt'
    ell,flip=0,0
    lat_0,lon_0=90.,0.
    fmt='pdf'
    sym,size='ro',8
    rsym,rsize='g^',8
    anti=0
    fancy=0
    coord=""
    if '-WD' in sys.argv:
        ind = sys.argv.index('-WD')
        dir_path=sys.argv[ind+1]
    if '-h' in sys.argv:
        print main.__doc__
        sys.exit()
    if '-S' in sys.argv:anti=1
    if '-fmt' in sys.argv:
        ind = sys.argv.index('-fmt')
        fmt=sys.argv[ind+1]
    if '-res' in sys.argv:
        ind = sys.argv.index('-res')
        res=sys.argv[ind+1]
    if '-etp' in sys.argv:fancy=1
    if '-prj' in sys.argv:
        ind = sys.argv.index('-prj')
        proj=sys.argv[ind+1]
    if '-rev' in sys.argv:
        flip=1
        ind = sys.argv.index('-rev')
        rsym=(sys.argv[ind+1])
        rsize=int(sys.argv[ind+2])
    if '-sym' in sys.argv:
        ind = sys.argv.index('-sym')
        sym=(sys.argv[ind+1])
        size=int(sys.argv[ind+2])
    if '-eye' in sys.argv:
        ind = sys.argv.index('-eye')
        lat_0=float(sys.argv[ind+1])
        lon_0=float(sys.argv[ind+2])
    if '-ell' in sys.argv: ell=1
    if '-age' in sys.argv: ages=1
    if '-f' in sys.argv:
        ind = sys.argv.index('-f')
        results_file=sys.argv[ind+1]
    if '-crd' in sys.argv:
        ind = sys.argv.index('-crd')
        crd=sys.argv[ind+1]
        if crd=='g':coord='0'
        if crd=='t':coord='100'
    results_file=dir_path+'/'+results_file
    data,file_type=pmag.magic_read(results_file)
    if file_type!='pmag_results':
        print "bad results file"
        sys.exit()
    FIG={'map':1}
    pmagplotlib.plot_init(FIG['map'],6,6)
    # read in er_sites file
    lats,lons,dp,dm,a95=[],[],[],[],[]
    Pars=[]
    dates,rlats,rlons=[],[],[]
    if 'data_type' in data[0].keys():
        Results=pmag.get_dictitem(data,'data_type','i','T') # get all site level data
    else:
        Results=data
    Results=pmag.get_dictitem(Results,'vgp_lat','','F') # get all non-blank latitudes
    Results=pmag.get_dictitem(Results,'vgp_lon','','F') # get all non-blank longitudes
    print Results
    if coord!="":Results=pmag.get_dictitem(Results,'tilt_correction',coord,'T') # get specified coordinate system
    for rec in Results:
            if 'average_age' in rec.keys() and rec['average_age']!="" and ages==1:
                dates.append(rec['average_age'])
            lat=float(rec['vgp_lat'])
            lon=float(rec['vgp_lon'])
            if flip==0:
                lats.append(lat)
                lons.append(lon)
            elif flip==1:
                if lat<0:
                    rlats.append(-lat)
                    lon=lon+180.
                    if lon>360:lon=lon-360.
                    rlons.append(lon)
                else:
                    lats.append(lat)
                    lons.append(lon)
            elif anti==1:
                lats.append(-lat)
                lon=lon+180.
                if lon>360:lon=lon-360.
                lons.append(lon)
            ppars=[]
            ppars.append(lon)
            ppars.append(lat)
            ell1,ell2="",""
            if 'vgp_dm' in rec.keys() and rec['vgp_dm']!="":ell1=float(rec['vgp_dm'])
            if 'vgp_dp' in rec.keys() and rec['vgp_dp']!="":ell2=float(rec['vgp_dp'])
            if 'vgp_alpha95' in rec.keys() and rec['vgp_alpha95']!="":ell1,ell2=float(rec['vgp_alpha95']),float(rec['vgp_alpha95'])
            if ell1!="" and ell2!="":
                ppars=[]
                ppars.append(lons[-1])
                ppars.append(lats[-1])
                ppars.append(ell1)
                ppars.append(lons[-1])
                isign=abs(lats[-1])/lats[-1]
                ppars.append(lats[-1]-isign*90.)
                ppars.append(ell2)
                ppars.append(lons[-1]+90.)
                ppars.append(0.)
                Pars.append(ppars)
    Opts={'latmin':-90,'latmax':90,'lonmin':0.,'lonmax':360.,'lat_0':lat_0,'lon_0':lon_0,'proj':proj,'sym':'bs','symsize':3,'pltgrid':0,'res':res,'boundinglat':0.}
    Opts['details']={'coasts':1,'rivers':0, 'states':0, 'countries':0,'ocean':1,'fancy':fancy}
    pmagplotlib.plotMAP(FIG['map'],[90.],[0.],Opts) # make the base map with a blue triangle at the pole`
    Opts['pltgrid']=-1
    Opts['sym']=sym
    Opts['symsize']=size
    if len(dates)>0:Opts['names']=dates
    if len(lats)>0:pmagplotlib.plotMAP(FIG['map'],lats,lons,Opts) # add the lats and lons of the poles
    Opts['names']=[]
    if len(rlats)>0:
        Opts['sym']=rsym
        Opts['symsize']=rsize
        pmagplotlib.plotMAP(FIG['map'],rlats,rlons,Opts) # add the lats and lons of the poles
    pmagplotlib.drawFIGS(FIG)
    if ell==1: # add ellipses if desired.
        Opts['details']={'coasts':0,'rivers':0, 'states':0, 'countries':0,'ocean':0}
        Opts['pltgrid']=-1 # turn off meridian replotting
        Opts['symsize']=2
        Opts['sym']='g-'
        for ppars in Pars:
            if ppars[2]!=0:
                PTS=pmagplotlib.plotELL(FIG['map'],ppars,'g.',0,0)
                elats,elons=[],[]
                for pt in PTS:
                    elons.append(pt[0])
                    elats.append(pt[1])
                pmagplotlib.plotMAP(FIG['map'],elats,elons,Opts) # make the base map with a blue triangle at the pole`
                pmagplotlib.drawFIGS(FIG)
    files={}
    for key in FIG.keys():
        files[key]='VGP_map'+'.'+fmt
    if pmagplotlib.isServer:
        black     = '#000000'
        purple    = '#800080'
        titles={}
        titles['eq']='VGP Map'
        FIG = pmagplotlib.addBorders(FIG,titles,black,purple)
        pmagplotlib.saveP(FIG,files)
    else:
        ans=raw_input(" S[a]ve to save plot, Return to quit:  ")
        if ans=="a":
            pmagplotlib.saveP(FIG,files)
        else:
            print "Good bye"
            sys.exit()
main()
<|endoftext|>


<commit_before>#############################
#
# Single Pulse cleaner
#
# Written by Daniele Michilli
#
#############################

import pandas as pd
import numpy as np
import os
import fnmatch
import matplotlib as mpl
mpl.use('Agg')
import multiprocessing as mp
import matplotlib.pyplot as plt
import logging
import re

import Events
import Pulses
import RFIexcision
import LSPplot
import Candidates
import Internet
from Parameters import *
from Paths import *

def obs_events(args):
  #----------------------------------------------------------
  # Creates the clean table for one observation and stores it
  #----------------------------------------------------------
  if args.conf: inc = 0
  else: inc = 12

  def file_list(folder, idL):
    file_list = []
    file_names = []
    for root, dirnames, filenames in os.walk(folder+idL):
      for filename in fnmatch.filter(filenames, '*singlepulse.tgz'):
        if filename in file_names:
          logging.warning("ATTENTION!: Two sp archives with the same filename foud: {}. Only the latter will be processed".format(filename))
          idx = file_names.index(filename)
          file_list[idx] = root+'/'+filename
          continue
        else:
          file_list.append(root+'/'+filename)
          file_names.append(filename)

    return file_list

  file_list = file_list(args.folder,args.idL)

  pulses = pulses_parallel(args.idL,file_list,args.folder)

  def merge_temp_databases(args.idL,store,file):
    store.append('events',pd.read_hdf('{}/{}'.format(TMP_FOLDER.format(args.idL),file),'events'),data_columns=['Pulse','SAP','BEAM','DM','Time'])
    meta_data = pd.read_hdf('{}/{}'.format(TMP_FOLDER.format(args.idL),file),'meta_data')
    meta_data.reset_index(inplace=True,drop=True)
    store.append('meta_data',meta_data)
    os.remove('{}/{}'.format(TMP_FOLDER.format(args.idL),file))

  store = pd.HDFStore('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'w')
  for file in os.listdir(TMP_FOLDER.format(args.idL)):
    if file.endswith('.tmp'):
      merge_temp_databases(args.idL,store,file)
  store.close()

  if pulses.empty:
    logging.warning("No pulse detected!")
    return
  pulses.sort_index(inplace=True)

  #TO BE TESTED

  '''
  # Nuove idee:
  # filtro 1 - rimuove pulses in cui, escludedno i beam vicini, ci sono piu di 1/10 di pulses in altri beam in dt, dDM
  # filtro 2 - rimuove pulses in cui ci sono piu' di 1/10 di beams aventi la meta' del SNR del beam con il massimo segnale entro dt, dDM
  # filtro 3 - raggruppa i pulses vicini in t,DM e applica statistiche simili agli eventi con i pulses. nel raggruppamento, se ci sono due pulses entro dt,dDM si raggruppa il piu' vicino

  #Remove time-spans affectd by RFI
  pulses.sort_index(inplace=True)
  pulses.Pulse.loc[RFIexcision.time_span(pulses[pulses.BEAM == inc])] += 1
  pulses.Pulse.loc[RFIexcision.time_span(pulses[pulses.BEAM > inc])] += 1
  pulses = pulses[pulses.Pulse <= RFI_percent]

  #Compare different beams
  pulses.Pulse.loc[RFIexcision.Compare_Beams(pulses[pulses.BEAM > inc].copy())] += 1
  pulses = pulses[pulses.Pulse <= RFI_percent]

  #Remove pulses appearing in too many beams
  pulses.Pulse += pulses.apply(lambda x: RFIexcision.puls_beams_select(x,pulses),axis=1).astype(np.int8)
  pulses = pulses[pulses.Pulse <= RFI_percent]
  '''

  pulses.Candidate = pulses.Candidate.astype(np.int32)
  cands = Candidates.candidates(pulses,args.idL)

  store = pd.HDFStore('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'a')
  store.append('pulses',pulses)
  if not cands.empty:
    cands.sort(['Sigma','Rank'],ascending=[0,1],inplace=True)
    store.append('candidates',cands)
    #cands = cands[cands.main_cand==0].head(30)
  store.close()

  if cands.empty: logging.warning("Any reliable candidate detected!")
  else:
    #Produce the output
    meta_data = pd.read_hdf('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'meta_data')
    LSPplot.output(args.idL, pulses, meta_data, cands, inc=inc)

    #Store the best candidates online
    try: Internet.upload(cands,args.idL,'{}/sp/candidates/.'.format(WRK_FOLDER.format(args.idL)))
    except:
      logging.exception("ATTENTION!\n\nConnession problem, update candidates in a second moment\n\n")
      with open('{}/{}/SP_ERROR.txt'.format(args.folder,args.args.idL),'a') as f:
        f.write("Connession problem \nConsider to run Upload.py script\n")

  return

def pulses_parallel(idL,dirs,folder):
  #Create events, meta_data and pulses lists
  CPUs = mp.cpu_count()
  dirs_range = int(np.ceil(len(dirs)/float(CPUs)))

  pool = mp.Pool(CPUs)
  results = [pool.apply_async(lists_creation, args=(idL,dirs[i*dirs_range:(i+1)*dirs_range],folder)) for i in range(CPUs) if len(dirs[i*dirs_range:(i+1)*dirs_range]) > 0]
  pool.close()
  pool.join()
  return pd.concat([p.get() for p in results])

def lists_creation(idL,dirs,folder):
  result = pd.DataFrame()

  for directory in dirs:
    def coord_from_path(directory):
      elements = os.path.basename(directory).split('_')
      sap = int(re.findall('\d', elements[1])[0])
      beam = int(re.findall('\d+', elements[2])[0])
      return sap, beam
    sap, beam = coord_from_path(directory)

    try:
      #Import the events
      events, meta_data = Events.Loader(directory,sap,beam)
      pulses = pd.DataFrame()

      if not events.empty:
        #Store the events
        events.to_hdf('{}/SAP{}_BEAM{}.tmp'.format(TMP_FOLDER.format(idL),sap,beam),'events',mode='w')
        meta_data.to_hdf('{}/SAP{}_BEAM{}.tmp'.format(TMP_FOLDER.format(idL),sap,beam),'meta_data',mode='a')

        pulses = pulses_from_events(events, idL, sap, beam)

    except:
      logging.warning("Some problem arised processing SAP "+str(sap)+" - BEAM "+str(beam)+", it will be discarded")
      with open('{}/{}/SP_ERROR.txt'.format(folder,idL),'a') as f:
        f.write("SAP {} - BEAM {} not processed due to some unknown error\n".format(sap, beam))
      pulses = pd.DataFrame()

    result = pd.concat((pulses,result))

  return result

def pulses_from_events(events, idL, sap, beam):
  #Correct for the time misalignment of events
  events.sort(['DM','Time'],inplace=True)  #Needed by TimeAlign
  events.Time = Events.TimeAlign(events.Time.copy(),events.DM)

  #Apply the thresholds to the events
  events = Events.Thresh(events)

  #Group the events
  events.sort(['DM','Time'],inplace=True) #Needed by Group
  Events.Group(events)
  events = events[events.Pulse>=0]

  #Generate the pulses
  pulses = Pulses.Generator(events)
  pulses = pulses[pulses.Sigma >= 6.5]

  #Set a maximum amout of pulses to prevent bad observations to block the pipeline
  pulses.sort('Sigma',ascending=False,inplace=True)
  pulses = pulses.iloc[:3e4]
  events = events[events.Pulse.isin(pulses.index)]

  #Apply RFI filters to the pulses
  pulses = RFIexcision.sift_pulses(pulses, events, idL, sap, beam)

  #A set of known pulses is necessary
  #Apply multimoment analysis to the pulses
  #events = events[events.Pulse.isin(pulses.index)]
  #RFIexcision.multimoment(pulses,idL)
  #pulses = pulses[pulses.Pulse <= RFI_percent]
  #events = events[events.Pulse.isin(pulses.index)]

  return pulses

<commit_msg>Bugs solved<commit_after>#############################
#
# Single Pulse cleaner
#
# Written by Daniele Michilli
#
#############################

import pandas as pd
import numpy as np
import os
import fnmatch
import matplotlib as mpl
mpl.use('Agg')
import multiprocessing as mp
import matplotlib.pyplot as plt
import logging
import re

import Events
import Pulses
import RFIexcision
import LSPplot
import Candidates
import Internet
from Parameters import *
from Paths import *

def obs_events(args):
  #----------------------------------------------------------
  # Creates the clean table for one observation and stores it
  #----------------------------------------------------------
  if args.conf: inc = 0
  else: inc = 12

  def file_list(folder, idL):
    file_list = []
    file_names = []
    for root, dirnames, filenames in os.walk(folder+idL):
      for filename in fnmatch.filter(filenames, '*singlepulse.tgz'):
        if filename in file_names:
          logging.warning("ATTENTION!: Two sp archives with the same filename foud: {}. Only the latter will be processed".format(filename))
          idx = file_names.index(filename)
          file_list[idx] = root+'/'+filename
          continue
        else:
          file_list.append(root+'/'+filename)
          file_names.append(filename)

    return file_list

  file_list = file_list(args.folder,args.idL)

  pulses = pulses_parallel(args.idL,file_list,args.folder)

  def merge_temp_databases(idL,store,file):
    store.append('events',pd.read_hdf('{}/{}'.format(TMP_FOLDER.format(idL),file),'events'),data_columns=['Pulse','SAP','BEAM','DM','Time'])
    meta_data = pd.read_hdf('{}/{}'.format(TMP_FOLDER.format(idL),file),'meta_data')
    meta_data.reset_index(inplace=True,drop=True)
    store.append('meta_data',meta_data)
    os.remove('{}/{}'.format(TMP_FOLDER.format(idL),file))

  store = pd.HDFStore('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'w')
  for file in os.listdir(TMP_FOLDER.format(args.idL)):
    if file.endswith('.tmp'):
      merge_temp_databases(args.idL,store,file)
  store.close()

  if pulses.empty:
    logging.warning("No pulse detected!")
    return
  pulses.sort_index(inplace=True)

  #TO BE TESTED

  '''
  # Nuove idee:
  # filtro 1 - rimuove pulses in cui, escludedno i beam vicini, ci sono piu di 1/10 di pulses in altri beam in dt, dDM
  # filtro 2 - rimuove pulses in cui ci sono piu' di 1/10 di beams aventi la meta' del SNR del beam con il massimo segnale entro dt, dDM
  # filtro 3 - raggruppa i pulses vicini in t,DM e applica statistiche simili agli eventi con i pulses. nel raggruppamento, se ci sono due pulses entro dt,dDM si raggruppa il piu' vicino

  #Remove time-spans affectd by RFI
  pulses.sort_index(inplace=True)
  pulses.Pulse.loc[RFIexcision.time_span(pulses[pulses.BEAM == inc])] += 1
  pulses.Pulse.loc[RFIexcision.time_span(pulses[pulses.BEAM > inc])] += 1
  pulses = pulses[pulses.Pulse <= RFI_percent]

  #Compare different beams
  pulses.Pulse.loc[RFIexcision.Compare_Beams(pulses[pulses.BEAM > inc].copy())] += 1
  pulses = pulses[pulses.Pulse <= RFI_percent]

  #Remove pulses appearing in too many beams
  pulses.Pulse += pulses.apply(lambda x: RFIexcision.puls_beams_select(x,pulses),axis=1).astype(np.int8)
  pulses = pulses[pulses.Pulse <= RFI_percent]
  '''

  pulses.Candidate = pulses.Candidate.astype(np.int32)
  cands = Candidates.candidates(pulses,args.idL)

  store = pd.HDFStore('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'a')
  store.append('pulses',pulses)
  if not cands.empty:
    cands.sort(['Sigma','Rank'],ascending=[0,1],inplace=True)
    store.append('candidates',cands)
    #cands = cands[cands.main_cand==0].head(30)
  store.close()

  if cands.empty: logging.warning("Any reliable candidate detected!")
  else:
    #Produce the output
    meta_data = pd.read_hdf('{}/sp/SinglePulses.hdf5'.format(WRK_FOLDER.format(args.idL)),'meta_data')
    LSPplot.output(args.idL, pulses, meta_data, cands, inc=inc)

    #Store the best candidates online
    try: Internet.upload(cands,args.idL,'{}/sp/candidates/.'.format(WRK_FOLDER.format(args.idL)))
    except:
      logging.exception("ATTENTION!\n\nConnession problem, update candidates in a second moment\n\n")
      with open('{}/{}/SP_ERROR.txt'.format(args.folder,args.args.idL),'a') as f:
        f.write("Connession problem \nConsider to run Upload.py script\n")

  return

def pulses_parallel(idL,dirs,folder):
  #Create events, meta_data and pulses lists
  CPUs = mp.cpu_count()
  dirs_range = int(np.ceil(len(dirs)/float(CPUs)))

  pool = mp.Pool(CPUs)
  results = [pool.apply_async(lists_creation, args=(idL,dirs[i*dirs_range:(i+1)*dirs_range],folder)) for i in range(CPUs) if len(dirs[i*dirs_range:(i+1)*dirs_range]) > 0]
  pool.close()
  pool.join()
  return pd.concat([p.get() for p in results])

def lists_creation(idL,dirs,folder):
  result = pd.DataFrame()

  for directory in dirs:
    def coord_from_path(directory):
      elements = os.path.basename(directory).split('_')
      sap = int(re.findall('\d', elements[1])[0])
      beam = int(re.findall('\d+', elements[2])[0])
      return sap, beam
    sap, beam = coord_from_path(directory)

    try:
      #Import the events
      events, meta_data = Events.Loader(directory,sap,beam)
      pulses = pd.DataFrame()

      if not events.empty:
        #Store the events
        events.to_hdf('{}/SAP{}_BEAM{}.tmp'.format(TMP_FOLDER.format(idL),sap,beam),'events',mode='w')
        meta_data.to_hdf('{}/SAP{}_BEAM{}.tmp'.format(TMP_FOLDER.format(idL),sap,beam),'meta_data',mode='a')

        pulses = pulses_from_events(events, idL, sap, beam)

    except:
      logging.warning("Some problem arised processing SAP "+str(sap)+" - BEAM "+str(beam)+", it will be discarded")
      with open('{}/{}/SP_ERROR.txt'.format(folder,idL),'a') as f:
        f.write("SAP {} - BEAM {} not processed due to some unknown error\n".format(sap, beam))
      pulses = pd.DataFrame()

    result = pd.concat((pulses,result))

  return result

def pulses_from_events(events, idL, sap, beam):
  #Correct for the time misalignment of events
  events.sort(['DM','Time'],inplace=True)  #Needed by TimeAlign
  events.Time = Events.TimeAlign(events.Time.copy(),events.DM)

  #Apply the thresholds to the events
  events = Events.Thresh(events)

  #Group the events
  events.sort(['DM','Time'],inplace=True) #Needed by Group
  Events.Group(events)
  events = events[events.Pulse>=0]

  #Generate the pulses
  pulses = Pulses.Generator(events)
  pulses = pulses[pulses.Sigma >= 6.5]

  #Set a maximum amout of pulses to prevent bad observations to block the pipeline
  pulses.sort('Sigma',ascending=False,inplace=True)
  pulses = pulses.iloc[:3e4]
  events = events[events.Pulse.isin(pulses.index)]

  #Apply RFI filters to the pulses
  pulses = RFIexcision.sift_pulses(pulses, events, idL, sap, beam)

  #A set of known pulses is necessary
  #Apply multimoment analysis to the pulses
  #events = events[events.Pulse.isin(pulses.index)]
  #RFIexcision.multimoment(pulses,idL)
  #pulses = pulses[pulses.Pulse <= RFI_percent]
  #events = events[events.Pulse.isin(pulses.index)]

  return pulses

<|endoftext|>