<commit_before>#!/usr/bin/env python

import numpy as np
import argparse
import os
import time

from SULI import which
from SULI.execute_command import execute_command
from SULI.work_within_directory import work_within_directory
from astropy.io import fits

if __name__ == "__main__":

    parser = argparse.ArgumentParser('Submit transient search to the farm at Stanford')

    # add the arguments needed to the parser
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--date', help='date specifying file to load')
    group.add_argument("--src_dir", help="Directory containing input data to be searched", type=str)

    parser.add_argument("--irf", help="Instrument response function name to be used", type=str, required=True)
    parser.add_argument("--probability", help="Probability of null hypothesis", type=float, required=True)
    parser.add_argument("--min_dist", help="Distance above which regions are not considered to overlap", type=float,
                        required=True)

    parser.add_argument("--res_dir", help="Directory where to put the results and logs for the search",
                        required=False, type=str, default=os.getcwd())
    parser.add_argument("--job_size", help="Number of jobs to submit at a time", required=False, type=int, default=20)
    parser.add_argument("--last_job", help="Integer specifying the last job submitted in this folder", required=False,
                        type=int, default=0)
    parser.add_argument('--test', dest='test_run', action='store_true')
    parser.set_defaults(test_run=False)

    # parse the arguments
    args = parser.parse_args()

    # get output directory from parser
    res_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.res_dir)))

    # Check that the output directory exists
    if not os.path.exists(res_dir):

        raise RuntimeError("Directory %s does not exist" % res_dir)

    # get src directory from parser
    src_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.src_dir)))

    # Check that the simulation directory exists
    if not os.path.exists(src_dir):

        raise RuntimeError("Directory %s does not exist" % src_dir)

    # Go to output directory
    with work_within_directory(res_dir):

        # Create logs directory if it does not exist
        if not os.path.exists('logs'):

            os.mkdir('logs')

        # Create generated_data directory if it does not exist, and get number of files in the directory
        if not os.path.exists('generated_data'):

            os.mkdir('generated_data')

        DIR = './generated_data'
        num_res_files = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, results))])

        # Generate universal command line parameters
        log_path = os.path.abspath('logs')
        out_path = os.path.abspath('generated_data')
        exe_path = which.which('search_on_farm.py')

        # if using simulated data:
        if args.src_dir:

            # get list of ft1 files
            ft1_files = [f for f in os.listdir(src_dir) if (str(os.path.join(src_dir, f)).endswith(('ft1.fits',
                                                                                'ft1.fit')))]
            # (is basically glob, should be changed to glob at some point)

            # get list of ft2 files
            ft2_files = [f for f in os.listdir(src_dir) if (str(os.path.join(src_dir, f)).endswith(('ft2.fits',
                                                                                'ft2.fit')))]

            # sort them
            def ft_sort(in_list):

                out_list = in_list.split("_")[1]

                return float(out_list)

            ft1_files = sorted(ft1_files, key=ft_sort)
            ft2_files = sorted(ft2_files, key=ft_sort)

            print '\nFound %s ft1 files\nFound %s ft2 files\n' % (len(ft1_files), len(ft2_files))

            # make sure each ft1/ft2 is part of a pair
            if len(ft1_files) != len(ft2_files):

                # determine which type there is more of for error msg
                if len(ft1_files) > len(ft2_files):

                    x = 'ft1 files'
                    y = 'ft2 files'

                else:

                    x = 'ft2 files'
                    y = 'ft1 files'

                raise RuntimeError('There are more %s than %s' % (x, y))

            # make sure pairs match
            for i in range(len(ft1_files)):

                with fits.open(os.path.join(src_dir, ft1_files[i])) as fits_file:

                    # Check the start and stop in the binary table
                    ft1_times = fits_file['EVENTS'].data.field("TIME")
                    ft1_starts = fits_file['GTI'].data.field("START")
                    ft1_stops = fits_file['GTI'].data.field("STOP")

                with fits.open(os.path.join(src_dir, ft2_files[i])) as fits_file:

                    # Check the start and stop in the binary table
                    ft2_starts = fits_file['SC_DATA'].data.field("START")
                    ft2_stops = fits_file['SC_DATA'].data.field("STOP")

                if ft2_starts.min() - min(ft1_starts.min(), ft1_times.min()) > 0:

                    raise RuntimeError("Mismatch in ft pair %s (FT2 file starts after the start of the FT1 file)" % i)

                if ft2_stops.max() - max(ft1_stops.max(), ft1_stops.max()) < 0:

                    raise RuntimeError("Mismatch in ft pair %s (FT2 file stops before the end of the FT1 file)" % i)

            # generate command line
            def sim_cmd_line(ft1, ft2, jobid):

                this_cmd_line = "qsub -l vmem=30gb -o %s/%s.out -e %s/%s.err -V -F '--inp_fts %s,%s --irf %s " \
                                "--probability %s --min_dist %s --out_dir %s' %s" % (log_path, jobid, log_path,
                                                                                jobid, ft1, ft2, args.irf,
                                                                                args.probability, args.min_dist,
                                                                                out_path, exe_path)
                return this_cmd_line

            # iterate over input directory, calling search on each pair of fits
            for i in range(args.last_job, len(ft1_files)):

                this_ft1 = src_dir + '/' + ft1_files[i]
                this_ft2 = src_dir + '/' + ft2_files[i]
                this_id = ft1_files[i]

                cmd_line = sim_cmd_line(this_ft1, this_ft2, this_id)
                if not args.test_run:

                    print "\nDay %s:" % (i + 1)
                    execute_command(cmd_line)

                # don't spam the farm; if more than [jobsize] jobs have been submitted,
                # wait until they finish to submit more
                if (i + 1) % args.job_size == 0:

                    # check res_dir every 10s for new results

                    # while the current number of results is not i+1 more than the initial number
                    # i.e., while the number of new files hasn't caught up to i + 1
                    num_fin = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR,
                                                                                results))])
                    sleep_count = 0
                    failed = False
                    while (num_fin - num_res_files) < (i + 1 - args.last_job) and failed is False:

                        # sleep for 30s
                        time.sleep(30)
                        sleep_count += 1

                        # update num_fin for any finished jobs
                        num_fin = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR,
                                                                                results))])
                        print "%s ouf of %s jobs in this pass finished." % ((num_fin - num_res_files) % args.job_size,
                                                                            args.job_size)

                        print "%s results in gen data (%s at start)" % (num_fin, num_res_files)

                        print "i = %s\n " \
                              "finished - initial = %s\n" \
                              "i+1-lastjob = %s\n" % (i, num_fin - num_res_files, i + 1 - args.last_job)

                        # some jobs may possibly fail
                        # if script has been on same batch for 15 min,
                        # check if there are log files (.out) for this batch in logs
                        # if so, batch is finished, move on
                        if sleep_count >= 30:

                            LOG = './logs'
                            num_out = len([out for out in os.listdir(LOG) if
                                           (str(os.path.join(LOG, out)).endswith('.out'))])

                            if num_out == i + 1:

                                failed = True

        else:

            def rl_cmd_line(start):

                this_cmd_line = "qsub -l vmem=10gb -o %s/%s.out -e %s/%s.err -V -F '--date %s --irf %s " \
                                "--probability %s --min_dist %s --out_dir %s' %s" % (log_path, start, log_path,
                                                                                start, start, args.irf,
                                                                                args.probability, args.min_dist,
                                                                                out_path, exe_path)
                return this_cmd_line

            # A year of Fermi data

            dates = np.arange(args.date, args.date + (365.0 * 86400.0), 86400.0)

            for this_tstart in dates:

                cmd_line = rl_cmd_line(this_tstart)

                if not args.test_run:

                    execute_command(cmd_line)
<commit_msg>changed dates parameter in submit to facilitate list of dates as input<commit_after>#!/usr/bin/env python

import numpy as np
import argparse
import os
import time

from SULI import which
from SULI.execute_command import execute_command
from SULI.work_within_directory import work_within_directory
from astropy.io import fits

if __name__ == "__main__":

    parser = argparse.ArgumentParser('Submit transient search to the farm at Stanford')

    # add the arguments needed to the parser
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--dates', help='Name of txt file containing dates to load', type=str)
    group.add_argument('--date', help='Date from which a year of data will be searched', type=int)
    group.add_argument("--src_dir", help="Directory containing input data to be searched", type=str)

    parser.add_argument("--irf", help="Instrument response function name to be used", type=str, required=True)
    parser.add_argument("--probability", help="Probability of null hypothesis", type=float, required=True)
    parser.add_argument("--min_dist", help="Distance above which regions are not considered to overlap", type=float,
                        required=True)

    parser.add_argument("--res_dir", help="Directory where to put the results and logs for the search",
                        required=False, type=str, default=os.getcwd())
    parser.add_argument("--job_size", help="Number of jobs to submit at a time", required=False, type=int, default=20)
    parser.add_argument("--last_job", help="Integer specifying the last job submitted in this folder", required=False,
                        type=int, default=0)
    parser.add_argument('--test', dest='test_run', action='store_true')
    parser.set_defaults(test_run=False)

    # parse the arguments
    args = parser.parse_args()

    # get output directory from parser
    res_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.res_dir)))

    # Check that the output directory exists
    if not os.path.exists(res_dir):

        raise RuntimeError("Directory %s does not exist" % res_dir)

    # get src directory from parser
    src_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.src_dir)))

    # Check that the simulation directory exists
    if not os.path.exists(src_dir):

        raise RuntimeError("Directory %s does not exist" % src_dir)

    # Go to output directory
    with work_within_directory(res_dir):

        # Create logs directory if it does not exist
        if not os.path.exists('logs'):

            os.mkdir('logs')

        # Create generated_data directory if it does not exist, and get number of files in the directory
        if not os.path.exists('generated_data'):

            os.mkdir('generated_data')

        DIR = './generated_data'
        num_res_files = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, results))])

        # Generate universal command line parameters
        log_path = os.path.abspath('logs')
        out_path = os.path.abspath('generated_data')
        exe_path = which.which('search_on_farm.py')

        # if using simulated data:
        if args.src_dir:

            # get list of ft1 files
            ft1_files = [f for f in os.listdir(src_dir) if (str(os.path.join(src_dir, f)).endswith(('ft1.fits',
                                                                                'ft1.fit')))]
            # (is basically glob, should be changed to glob at some point)

            # get list of ft2 files
            ft2_files = [f for f in os.listdir(src_dir) if (str(os.path.join(src_dir, f)).endswith(('ft2.fits',
                                                                                'ft2.fit')))]

            # sort them
            def ft_sort(in_list):

                out_list = in_list.split("_")[1]

                return float(out_list)

            ft1_files = sorted(ft1_files, key=ft_sort)
            ft2_files = sorted(ft2_files, key=ft_sort)

            print '\nFound %s ft1 files\nFound %s ft2 files\n' % (len(ft1_files), len(ft2_files))

            # make sure each ft1/ft2 is part of a pair
            if len(ft1_files) != len(ft2_files):

                # determine which type there is more of for error msg
                if len(ft1_files) > len(ft2_files):

                    x = 'ft1 files'
                    y = 'ft2 files'

                else:

                    x = 'ft2 files'
                    y = 'ft1 files'

                raise RuntimeError('There are more %s than %s' % (x, y))

            # make sure pairs match
            for i in range(len(ft1_files)):

                with fits.open(os.path.join(src_dir, ft1_files[i])) as fits_file:

                    # Check the start and stop in the binary table
                    ft1_times = fits_file['EVENTS'].data.field("TIME")
                    ft1_starts = fits_file['GTI'].data.field("START")
                    ft1_stops = fits_file['GTI'].data.field("STOP")

                with fits.open(os.path.join(src_dir, ft2_files[i])) as fits_file:

                    # Check the start and stop in the binary table
                    ft2_starts = fits_file['SC_DATA'].data.field("START")
                    ft2_stops = fits_file['SC_DATA'].data.field("STOP")

                if ft2_starts.min() - min(ft1_starts.min(), ft1_times.min()) > 0:

                    raise RuntimeError("Mismatch in ft pair %s (FT2 file starts after the start of the FT1 file)" % i)

                if ft2_stops.max() - max(ft1_stops.max(), ft1_stops.max()) < 0:

                    raise RuntimeError("Mismatch in ft pair %s (FT2 file stops before the end of the FT1 file)" % i)

            # generate command line
            def sim_cmd_line(ft1, ft2, jobid):

                this_cmd_line = "qsub -l vmem=30gb -o %s/%s.out -e %s/%s.err -V -F '--inp_fts %s,%s --irf %s " \
                                "--probability %s --min_dist %s --out_dir %s' %s" % (log_path, jobid, log_path,
                                                                                jobid, ft1, ft2, args.irf,
                                                                                args.probability, args.min_dist,
                                                                                out_path, exe_path)
                return this_cmd_line

            # iterate over input directory, calling search on each pair of fits
            for i in range(args.last_job, len(ft1_files)):

                this_ft1 = src_dir + '/' + ft1_files[i]
                this_ft2 = src_dir + '/' + ft2_files[i]
                this_id = ft1_files[i]

                cmd_line = sim_cmd_line(this_ft1, this_ft2, this_id)
                if not args.test_run:

                    print "\nDay %s:" % (i + 1)
                    execute_command(cmd_line)

                # don't spam the farm; if more than [jobsize] jobs have been submitted,
                # wait until they finish to submit more
                if (i + 1) % args.job_size == 0:

                    # check res_dir every 10s for new results

                    # while the current number of results is not i+1 more than the initial number
                    # i.e., while the number of new files hasn't caught up to i + 1
                    num_fin = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR,
                                                                                results))])
                    sleep_count = 0
                    failed = False
                    while (num_fin - num_res_files) < (i + 1 - args.last_job) and failed is False:

                        # sleep for 30s
                        time.sleep(30)
                        sleep_count += 1

                        # update num_fin for any finished jobs
                        num_fin = len([results for results in os.listdir(DIR) if os.path.isfile(os.path.join(DIR,
                                                                                results))])
                        print "%s ouf of %s jobs in this pass finished." % ((num_fin - num_res_files) % args.job_size,
                                                                            args.job_size)

                        print "%s results in gen data (%s at start)" % (num_fin, num_res_files)

                        print "i = %s\n " \
                              "finished - initial = %s\n" \
                              "i+1-lastjob = %s\n" % (i, num_fin - num_res_files, i + 1 - args.last_job)

                        # some jobs may possibly fail
                        # if script has been on same batch for 15 min,
                        # check if there are log files (.out) for this batch in logs
                        # if so, batch is finished, move on
                        if sleep_count >= 30:

                            LOG = './logs'
                            num_out = len([out for out in os.listdir(LOG) if
                                           (str(os.path.join(LOG, out)).endswith('.out'))])

                            if num_out == i + 1:

                                failed = True

        else:

            def rl_cmd_line(start):

                this_cmd_line = "qsub -l vmem=10gb -o %s/%s.out -e %s/%s.err -V -F '--date %s --irf %s " \
                                "--probability %s --min_dist %s --out_dir %s' %s" % (log_path, start, log_path,
                                                                                start, start, args.irf,
                                                                                args.probability, args.min_dist,
                                                                                out_path, exe_path)
                return this_cmd_line

            # A year of Fermi data
            if args.date:

                dates = np.arange(args.date, args.date + (365.0 * 86400.0), 86400.0)

                for this_tstart in dates:

                    cmd_line = rl_cmd_line(this_tstart)

                    if not args.test_run:

                        execute_command(cmd_line)

            # A list of dates
            else:

                # get dates from file as a list
                dates = [line.rstrip('\n') for line in open(args.dates)]

                for i in range(len(dates)):

                    cmd_line = rl_cmd_line(dates[i])

                    if not args.test_run:

                        execute_command(cmd_line)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
#   Spack Project Developers. See the top-level COPYRIGHT file for details.
# Copyright 2020 GSI Helmholtz Centre for Heavy Ion Research GmbH,
#   Darmstadt, Germany
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Fairlogger(CMakePackage):
    """Lightweight and fast C++ Logging Library"""

    homepage = 'https://github.com/FairRootGroup/FairLogger'
    url = "https://github.com/FairRootGroup/FairLogger/archive/v1.2.0.tar.gz"
    git = 'https://github.com/FairRootGroup/FairLogger.git'
    maintainers = ['dennisklein', 'ChristianTackeGSI']
    # generator = 'Ninja'

    version('develop', branch='dev')
    version('1.6.1', sha256='3894580f4c398d724ba408e410e50f70c9f452e8cfaf7c3ff8118c08df28eaa8')
    version('1.6.0', sha256='721e8cadfceb2f63014c2a727e098babc6deba653baab8866445a772385d0f5b')
    version('1.5.0', sha256='8e74e0b1e50ee86f4fca87a44c6b393740b32099ac3880046bf252c31c58dd42')
    version('1.4.0', sha256='75457e86984cc03ce87d6ad37adc5aab1910cabd39a9bbe5fb21ce2475a91138')
    version('1.3.0', sha256='5cedea2773f7091d69aae9fd8f724e6e47929ee3784acdd295945a848eb36b93')
    version('1.2.0', sha256='bc0e049cf84ceb308132d8679e7f22fcdca5561dda314d5233d0d5fe2b0f8c62')
    version('1.1.0', sha256='e185e5bd07df648224f85e765d18579fae0de54adaab9a194335e3ad6d3d29f7')
    version('1.0.6', sha256='2fc266a6e494adda40837be406aef8d9838f385ffd64fbfafb1164833906b4e0')

    variant('build_type', default='RelWithDebInfo',
            values=('Debug', 'Release', 'RelWithDebInfo'),
            multi=False,
            description='CMake build type')
    variant('cxxstd', default='11',
            values=('11', '14', '17'),
            multi=False,
            description='Use the specified C++ standard when building.')
    variant('pretty',
            default=False,
            description='Use BOOST_PRETTY_FUNCTION macro.')
    conflicts('+pretty', when='@:1.3.99')

    depends_on('cmake@3.9.4:', type='build')
    depends_on('git', type='build', when='@develop')
    depends_on('boost', when='+pretty')
    depends_on('fmt@5.3.0:', when='@1.6.0:')

    def patch(self):
        """FairLogger gets its version number from git.
           But the tarball doesn't have that information, so
           we patch the spack version into CMakeLists.txt"""
        if not self.spec.satisfies("@develop"):
            filter_file(r'(get_git_version\(.*)\)',
                        r'\1 DEFAULT_VERSION %s)' % self.spec.version,
                        'CMakeLists.txt')

    def cmake_args(self):
        cxxstd = self.spec.variants['cxxstd'].value
        args = []
        args.append('-DCMAKE_CXX_STANDARD=%s' % cxxstd)
        args.append('-DUSE_BOOST_PRETTY_FUNCTION=%s' %
                    ('ON' if '+pretty' in self.spec else 'OFF'))
        if self.spec.satisfies('@1.6:'):
            args.append('-DUSE_EXTERNAL_FMT=ON')
        return args
<commit_msg>fairlogger: add version, other improvements (#15300)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
#   Spack Project Developers. See the top-level COPYRIGHT file for details.
# Copyright 2020 GSI Helmholtz Centre for Heavy Ion Research GmbH,
#   Darmstadt, Germany
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Fairlogger(CMakePackage):
    """Lightweight and fast C++ Logging Library"""

    homepage = 'https://github.com/FairRootGroup/FairLogger'
    url = "https://github.com/FairRootGroup/FairLogger/archive/v1.2.0.tar.gz"
    git = 'https://github.com/FairRootGroup/FairLogger.git'
    maintainers = ['dennisklein', 'ChristianTackeGSI']
    # generator = 'Ninja'

    version('develop', branch='dev', get_full_repo=True)
    version('1.6.2', sha256='5c6ef0c0029eb451fee71756cb96e6c5011040a9813e8889667b6f3b6b04ed03')
    version('1.6.1', sha256='3894580f4c398d724ba408e410e50f70c9f452e8cfaf7c3ff8118c08df28eaa8')
    version('1.6.0', sha256='721e8cadfceb2f63014c2a727e098babc6deba653baab8866445a772385d0f5b')
    version('1.5.0', sha256='8e74e0b1e50ee86f4fca87a44c6b393740b32099ac3880046bf252c31c58dd42')
    version('1.4.0', sha256='75457e86984cc03ce87d6ad37adc5aab1910cabd39a9bbe5fb21ce2475a91138')
    version('1.3.0', sha256='5cedea2773f7091d69aae9fd8f724e6e47929ee3784acdd295945a848eb36b93')
    version('1.2.0', sha256='bc0e049cf84ceb308132d8679e7f22fcdca5561dda314d5233d0d5fe2b0f8c62')
    version('1.1.0', sha256='e185e5bd07df648224f85e765d18579fae0de54adaab9a194335e3ad6d3d29f7')
    version('1.0.6', sha256='2fc266a6e494adda40837be406aef8d9838f385ffd64fbfafb1164833906b4e0')

    variant('build_type', default='RelWithDebInfo',
            values=('Debug', 'Release', 'RelWithDebInfo'),
            multi=False,
            description='CMake build type')
    variant('cxxstd', default='default',
            values=('11', '14', '17'),
            multi=False,
            description='Use the specified C++ standard when building.')
    variant('pretty',
            default=False,
            description='Use BOOST_PRETTY_FUNCTION macro (Supported by 1.4+).')
    conflicts('+pretty', when='@:1.3.99')

    depends_on('cmake@3.9.4:', type='build')
    depends_on('git', type='build', when='@develop')

    depends_on('boost', when='+pretty')
    conflicts('^boost@1.70:', when='^cmake@:3.14')
    depends_on('fmt@5.3.0:5.99', when='@1.6.0:1.6.1')
    depends_on('fmt@5.3.0:', when='@1.6.2:')

    def patch(self):
        """FairLogger gets its version number from git.
           But the tarball doesn't have that information, so
           we patch the spack version into CMakeLists.txt"""
        if not self.spec.satisfies("@develop"):
            filter_file(r'(get_git_version\(.*)\)',
                        r'\1 DEFAULT_VERSION %s)' % self.spec.version,
                        'CMakeLists.txt')

    def cmake_args(self):
        args = []
        cxxstd = self.spec.variants['cxxstd'].value
        if cxxstd != 'default':
            args.append('-DCMAKE_CXX_STANDARD=%s' % cxxstd)
        args.append('-DUSE_BOOST_PRETTY_FUNCTION=%s' %
                    ('ON' if '+pretty' in self.spec else 'OFF'))
        if self.spec.satisfies('@1.6:'):
            args.append('-DUSE_EXTERNAL_FMT=ON')
        return args
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""urls.py: messages extends"""

from django.conf.urls import patterns, url

urlpatterns = patterns('',
    url(r'^mark_read/(?P<message_id>\d+)/$', 'messages_extends.views.message_mark_read', name='message_mark_read'),
    url(r'^mark_read/all/$', 'messages_extends.views.message_mark_all_read', name='message_mark_all_read'),
)
<commit_msg>String to view import (Avoid deprecated warning)<commit_after># -*- coding: utf-8 -*-
"""urls.py: messages extends"""

from django.conf.urls import url
from .views import message_mark_all_read, message_mark_read
urlpatterns = [
    url(r'^mark_read/(?P<message_id>\d+)/$', message_mark_read, name='message_mark_read'),
    url(r'^mark_read/all/$', message_mark_all_read, name='message_mark_all_read'),
]
<|endoftext|>


<commit_before>##############################################################################
#                                                                            #
#  This is a demonstration of the Fast Factorized Backprojection algorithm.  #
#  Data sets can be switched in and out by commenting/uncommenting the lines #
#  of code below.                                                            #
#                                                                            #
##############################################################################

#Add include directories to default path list
from sys import path
path.append('../')
path.append('./dictionaries')

#Include Dictionaries
from SARplatform import plat_dict

#Include standard library dependencies
import matplotlib.pylab as plt
from time import time

#Include SARIT toolset
from ritsar import phsTools
from ritsar import phsRead
from ritsar import imgTools

if __name__ == '__main__':
    '''
    #simulated FFBP demo
    ##############################################################################
    #Create platform dictionary
    platform = plat_dict()

    #Create image plane dictionary
    img_plane = imgTools.img_plane_dict(platform, aspect = 1, res_factor=0.9)

    #Simulate phase history
    nsamples = platform['nsamples']
    npulses = platform['npulses']
    x = img_plane['u']; y = img_plane['v']
    points = [[0,0,0],
              [200,0,0],
              [0,100,0]]
    amplitudes = [1,1,1]
    phs = phsTools.simulate_phs(platform, points, amplitudes)

    #Apply RVP correction
    phs = phsTools.RVP_correct(phs, platform)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()
    '''
    #AFRL DSBP demo
    ###############################################################################
    #Define top level directory containing *.mat file
    #and choose polarization and starting azimuth
    pol = 'HH'
    directory = './data/AFRL/pass1'
    start_az = 1

    #Import phase history and create platform dictionary
    [phs, platform] = phsRead.AFRL(directory, pol, start_az, n_az = 4)

    #Create image plane dictionary
    img_plane = imgTools.img_plane_dict(platform, res_factor = 1.0, upsample = True, aspect = 1.0)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()
    '''
    #DIRSIG DSBP demo
    ###############################################################################
    #Define directory containing *.au2 and *.phs files
    directory = './data/DIRSIG/'

    #Import phase history and create platform dictionary
    [phs, platform] = phsRead.DIRSIG(directory)

    #Correct for reisdual video phase
    phs_corr = phsTools.RVP_correct(phs, platform)

    #Import image plane dictionary from './parameters/img_plane'
    img_plane = imgTools.img_plane_dict(platform, res_factor = 1.0, aspect = 1.0)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()'''
<commit_msg>Update FFBPmp_demo.py<commit_after>##############################################################################
#                                                                            #
#  This is a demonstration of the Fast Factorized Backprojection algorithm.  #
#  Data sets can be switched in and out by commenting/uncommenting the lines #
#  of code below.                                                            #
#                                                                            #
##############################################################################

#Add include directories to default path list
from sys import path
path.append('../')
path.append('./dictionaries')

#Include Dictionaries
from SARplatform import plat_dict

#Include standard library dependencies
import matplotlib.pylab as plt
from time import time

#Include SARIT toolset
from ritsar import phsTools
from ritsar import phsRead
from ritsar import imgTools

if __name__ == '__main__':
    '''
    #simulated FFBP demo
    ##############################################################################
    #Create platform dictionary
    platform = plat_dict()

    #Create image plane dictionary
    img_plane = imgTools.img_plane_dict(platform, aspect = 1, res_factor=0.9)

    #Simulate phase history
    nsamples = platform['nsamples']
    npulses = platform['npulses']
    x = img_plane['u']; y = img_plane['v']
    points = [[0,0,0],
              [200,0,0],
              [0,100,0]]
    amplitudes = [1,1,1]
    phs = phsTools.simulate_phs(platform, points, amplitudes)

    #Apply RVP correction
    phs = phsTools.RVP_correct(phs, platform)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 4)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 4)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()
    '''
    #AFRL DSBP demo
    ###############################################################################
    #Define top level directory containing *.mat file
    #and choose polarization and starting azimuth
    pol = 'HH'
    directory = './data/AFRL/pass1'
    start_az = 1

    #Import phase history and create platform dictionary
    [phs, platform] = phsRead.AFRL(directory, pol, start_az, n_az = 4)

    #Create image plane dictionary
    img_plane = imgTools.img_plane_dict(platform, res_factor = 1.0, upsample = True, aspect = 1.0)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 2)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-30,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()
    '''
    #DIRSIG DSBP demo
    ###############################################################################
    #Define directory containing *.au2 and *.phs files
    directory = './data/DIRSIG/'

    #Import phase history and create platform dictionary
    [phs, platform] = phsRead.DIRSIG(directory)

    #Correct for reisdual video phase
    phs_corr = phsTools.RVP_correct(phs, platform)

    #Import image plane dictionary from './parameters/img_plane'
    img_plane = imgTools.img_plane_dict(platform, res_factor = 1.0, aspect = 1.0)

    #full backprojection
    start = time()
    img_bp   = imgTools.backprojection(phs, platform, img_plane, taylor = 17, upsample = 2)
    bp_time = time()-start

    #Fast-factorized backprojection without multi-processing
    start = time()
    img_FFBP = imgTools.FFBP(phs, platform, img_plane, taylor = 17, factor_max = 4)
    fbp_time = time()-start

    #Fast-factorized backprojection with multi-processing
    start = time()
    img_FFBP = imgTools.FFBPmp(phs, platform, img_plane, taylor = 17, factor_max = 4)
    fbpmp_time = time()-start

    #Output image
    u = img_plane['u']; v = img_plane['v']
    extent = [u.min(), u.max(), v.min(), v.max()]

    plt.subplot(2,1,1)
    plt.title('Full Backprojection \n \
    Runtime = %i s'%bp_time)
    imgTools.imshow(img_bp, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')

    plt.subplot(2,2,3)
    plt.title('Fast Factorized Backprojection \n w/0 multi-processing \n \
    Runtime = %i s'%fbp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()

    plt.subplot(2,2,4)
    plt.title('Fast Factorized Backprojection \n w/ multi=processing \n \
    Runtime = %i s'%fbpmp_time)
    imgTools.imshow(img_FFBP, dB_scale = [-25,0], extent = extent)
    plt.xlabel('meters'); plt.ylabel('meters')
    plt.tight_layout()'''
<|endoftext|>


<commit_before>from portality.core import app
from lxml import etree
import re
from portality.bll import exceptions
from portality.crosswalks.exceptions import CrosswalkException
from portality import models
from datetime import datetime

class DOAJXWalk(object):
    format_name = "doaj"
    schema_path = app.config.get("SCHEMAS", {}).get("doaj")

    def __init__(self):
        self.validation_log = ""

        # load the schema into memory for more efficient usage in repeat calls to the crosswalk
        if self.schema_path is None:
            raise exceptions.IngestException(message="Unable to validate for DOAJXWalk, as schema path is not set in config")

        try:
            schema_file = open(self.schema_path)
            schema_doc = etree.parse(schema_file)
            self.schema = etree.XMLSchema(schema_doc)
        except Exception as e:
            raise exceptions.IngestException(message="There was an error attempting to load schema from " + self.schema_path, inner=e)

    def validate_file(self, file_handle):
        # first try to parse the file
        try:
            doc = etree.parse(file_handle)
        except etree.XMLSyntaxError as e:   # although the treatment is the same, pulling this out so we remember what the primary kind of exception should be
            raise CrosswalkException(message="Unable to parse XML file", inner=e)
        except Exception as e:
            raise CrosswalkException(message="Unable to parse XML file", inner=e)

        # then pass the doc to the validator
        valid = self.validate(doc)

        if not valid:
            msg = "Validation message from schema '{x}': {y}\n".format(x=DOAJXWalk.format_name, y=self.validation_log)
            raise CrosswalkException(message="Unable to validate document with identified schema", inner_message=msg)

        return doc

    def validate(self, doc):
        valid = self.schema.validate(doc)
        if not valid:
            el = self.schema.error_log.__repr__()
            # strip the filename, as we don't want to leak the path to the UI
            rx = "[\da-f]{32}.xml:(.*)"
            match = re.search(rx, el)
            if match is not None:
                el = match.group(1)
            self.validation_log = el
        return valid

    def crosswalk_file(self, file_handle, add_journal_info=True):
        doc = self.validate_file(file_handle)
        return self.crosswalk_doc(doc, add_journal_info=add_journal_info)

    def crosswalk_doc(self, doc, add_journal_info=True):
        # go through the records in the doc and crosswalk each one individually
        articles = []
        root = doc.getroot()
        for record in root.findall("record"):
            article = self.crosswalk_article(record, add_journal_info=add_journal_info)
            articles.append(article)

        return articles

    def crosswalk_article(self, record, add_journal_info=True):
        """
        Example record:
        <record>
          <language>eng</language>
          <publisher>Co-Action Publishing</publisher>
          <journalTitle>Tellus A</journalTitle>
          <issn>0280-6495</issn>
          <eissn>1600-0870</eissn>
          <publicationDate>2014-02-05</publicationDate>
          <volume>66</volume>
          <issue>0</issue>
          <startPage>1</startPage>
          <endPage>18</endPage>
          <doi>10.3402/tellusa.v66.21390</doi>
          <publisherRecordId>21390</publisherRecordId>
          <documentType>Original</documentType>
          <title language="eng">LakeMIP Kivu...</title>
          <authors>
             <author>
                <name>WIM Thiery</name>
                <affiliationId>1</affiliationId>
            </author>
         </authors>
          <affiliationsList>
             <affiliationName affiliationId="1">
		            Department of Earth and Environmental Sciences...</affiliationName>
         </affiliationsList>

          <abstract language="eng">The African great...</abstract>
          <fullTextUrl format="pdf">http://www.tellusa.net/index.php/tellusa/article/download/21390/pdf_1</fullTextUrl>
          <keywords language="eng">
             <keyword>lake modelling</keyword>
         </keywords>
        </record>
        """
        article = models.Article()
        bibjson = article.bibjson()

        # language
        lang = _element(record, "language")
        if lang is not None:
            bibjson.journal_language = lang

        # publisher
        pub = _element(record, "publisher")
        if pub is not None:
            bibjson.publisher = pub

        # journal title
        jt = _element(record, "journalTitle")
        if jt is not None:
            bibjson.journal_title = jt

        # p-issn
        pissn = _element(record, "issn")
        if pissn is not None:
            bibjson.add_identifier(bibjson.P_ISSN, pissn.upper())

        # e-issn
        eissn = _element(record, "eissn")
        if eissn is not None:
            bibjson.add_identifier(bibjson.E_ISSN, eissn.upper())

        # publication date
        pd = _element(record, "publicationDate")
        if pd is not None:
            y, m = _year_month(pd)
            if y is not None:
                bibjson.year = y
            if m is not None:
                bibjson.month = m

        # volume
        vol = _element(record, "volume")
        if vol is not None:
            bibjson.volume = vol

        # issue
        iss = _element(record, "issue")
        if iss is not None:
            bibjson.number = iss

        # start page
        sp = _element(record, "startPage")
        if sp is not None:
            bibjson.start_page = sp

        # end page
        ep = _element(record, "endPage")
        if ep is not None:
            bibjson.end_page = ep

        # doi
        doi = _element(record, "doi")
        if doi is not None:
            bibjson.add_identifier(bibjson.DOI, doi)

        # publisher record id
        pri = _element(record, "publisherRecordId")
        if pri is not None:
            article.set_publisher_record_id(pri)

        # document type
        dt = _element(record, "documentType")
        if dt is not None:
            # FIXME: outstanding question as to what to do with this
            pass

        # title
        title = _element(record, "title")
        if title is not None:
            bibjson.title = title

        # authors
        ## first we need to extract the affiliations
        affiliations = {}
        affel = record.find("affiliationsList")
        if affel is not None:
            for ael in affel:
                affid = ael.get("affiliationId")
                aff = ael.text
                affiliations[affid] = aff
        ## now crosswalk each author and dereference their affiliation from the table
        authorsel = record.find("authors")
        if authorsel is not None:
            for ael in authorsel:
                name = _element(ael, "name")
                affid = _element(ael, "affiliationId")
                aff = affiliations.get(affid)
                bibjson.add_author(name, affiliation=aff)

        # abstract
        abstract = _element(record, "abstract")
        if abstract is not None:
            bibjson.abstract = abstract[:30000]  # avoids Elasticsearch
            # exceptions about .exact analyser not being able to handle
            # more than 32766 UTF8 characters

        # fulltext
        ftel = record.find("fullTextUrl")
        if ftel is not None and ftel.text is not None and ftel.text != "":
            ct = ftel.get("format")
            url = ftel.text
            bibjson.add_url(url, "fulltext", ct)

        # keywords
        keyel = record.find("keywords")
        if keyel is not None:
            for kel in keyel:
                if kel.text != "":
                    bibjson.add_keyword(kel.text)

        # add the journal info if requested
        if add_journal_info:
            article.add_journal_metadata()

        return article

###############################################################################
## some convenient utilities
###############################################################################

def _year_month(date):
    try:
        stamp = datetime.strptime(date, "%Y-%m-%d")
        return stamp.year, stamp.month
    except:
        pass
    try:
        stamp = datetime.strptime(date, "%Y-%m")
        return stamp.year, stamp.month
    except:
        pass
    try:
        stamp = datetime.strptime(date, "%Y")
        return stamp.year, None
    except:
        pass
    return None, None

def _element(xml, field):
    el = xml.find(field)
    if el is not None and el.text is not None and el.text != "":
        return el.text.strip()
    return None<commit_msg>add url validation to file uploader<commit_after>from portality.core import app
from lxml import etree
import re
from portality.bll import exceptions
from portality.crosswalks.exceptions import CrosswalkException
from portality import models
from datetime import datetime
from urlparse import urlparse

class DOAJXWalk(object):
    format_name = "doaj"
    schema_path = app.config.get("SCHEMAS", {}).get("doaj")

    def __init__(self):
        self.validation_log = ""

        # load the schema into memory for more efficient usage in repeat calls to the crosswalk
        if self.schema_path is None:
            raise exceptions.IngestException(message="Unable to validate for DOAJXWalk, as schema path is not set in config")

        try:
            schema_file = open(self.schema_path)
            schema_doc = etree.parse(schema_file)
            self.schema = etree.XMLSchema(schema_doc)
        except Exception as e:
            raise exceptions.IngestException(message="There was an error attempting to load schema from " + self.schema_path, inner=e)

    def validate_file(self, file_handle):
        # first try to parse the file
        try:
            doc = etree.parse(file_handle)
        except etree.XMLSyntaxError as e:   # although the treatment is the same, pulling this out so we remember what the primary kind of exception should be
            raise CrosswalkException(message="Unable to parse XML file", inner=e)
        except Exception as e:
            raise CrosswalkException(message="Unable to parse XML file", inner=e)

        # then pass the doc to the validator
        valid = self.validate(doc)

        if not valid:
            msg = "Validation message from schema '{x}': {y}\n".format(x=DOAJXWalk.format_name, y=self.validation_log)
            raise CrosswalkException(message="Unable to validate document with identified schema", inner_message=msg)

        return doc

    def validate(self, doc):
        valid = self.schema.validate(doc)
        if not valid:
            el = self.schema.error_log.__repr__()
            # strip the filename, as we don't want to leak the path to the UI
            rx = "[\da-f]{32}.xml:(.*)"
            match = re.search(rx, el)
            if match is not None:
                el = match.group(1)
            self.validation_log = el
        return valid

    def crosswalk_file(self, file_handle, add_journal_info=True):
        doc = self.validate_file(file_handle)
        return self.crosswalk_doc(doc, add_journal_info=add_journal_info)

    def crosswalk_doc(self, doc, add_journal_info=True):
        # go through the records in the doc and crosswalk each one individually
        articles = []
        root = doc.getroot()
        for record in root.findall("record"):
            article = self.crosswalk_article(record, add_journal_info=add_journal_info)
            articles.append(article)

        return articles

    def crosswalk_article(self, record, add_journal_info=True):
        """
        Example record:
        <record>
          <language>eng</language>
          <publisher>Co-Action Publishing</publisher>
          <journalTitle>Tellus A</journalTitle>
          <issn>0280-6495</issn>
          <eissn>1600-0870</eissn>
          <publicationDate>2014-02-05</publicationDate>
          <volume>66</volume>
          <issue>0</issue>
          <startPage>1</startPage>
          <endPage>18</endPage>
          <doi>10.3402/tellusa.v66.21390</doi>
          <publisherRecordId>21390</publisherRecordId>
          <documentType>Original</documentType>
          <title language="eng">LakeMIP Kivu...</title>
          <authors>
             <author>
                <name>WIM Thiery</name>
                <affiliationId>1</affiliationId>
            </author>
         </authors>
          <affiliationsList>
             <affiliationName affiliationId="1">
		            Department of Earth and Environmental Sciences...</affiliationName>
         </affiliationsList>

          <abstract language="eng">The African great...</abstract>
          <fullTextUrl format="pdf">http://www.tellusa.net/index.php/tellusa/article/download/21390/pdf_1</fullTextUrl>
          <keywords language="eng">
             <keyword>lake modelling</keyword>
         </keywords>
        </record>
        """
        article = models.Article()
        bibjson = article.bibjson()

        # language
        lang = _element(record, "language")
        if lang is not None:
            bibjson.journal_language = lang

        # publisher
        pub = _element(record, "publisher")
        if pub is not None:
            bibjson.publisher = pub

        # journal title
        jt = _element(record, "journalTitle")
        if jt is not None:
            bibjson.journal_title = jt

        # p-issn
        pissn = _element(record, "issn")
        if pissn is not None:
            bibjson.add_identifier(bibjson.P_ISSN, pissn.upper())

        # e-issn
        eissn = _element(record, "eissn")
        if eissn is not None:
            bibjson.add_identifier(bibjson.E_ISSN, eissn.upper())

        # publication date
        pd = _element(record, "publicationDate")
        if pd is not None:
            y, m = _year_month(pd)
            if y is not None:
                bibjson.year = y
            if m is not None:
                bibjson.month = m

        # volume
        vol = _element(record, "volume")
        if vol is not None:
            bibjson.volume = vol

        # issue
        iss = _element(record, "issue")
        if iss is not None:
            bibjson.number = iss

        # start page
        sp = _element(record, "startPage")
        if sp is not None:
            bibjson.start_page = sp

        # end page
        ep = _element(record, "endPage")
        if ep is not None:
            bibjson.end_page = ep

        # doi
        doi = _element(record, "doi")
        if doi is not None:
            bibjson.add_identifier(bibjson.DOI, doi)

        # publisher record id
        pri = _element(record, "publisherRecordId")
        if pri is not None:
            article.set_publisher_record_id(pri)

        # document type
        dt = _element(record, "documentType")
        if dt is not None:
            # FIXME: outstanding question as to what to do with this
            pass

        # title
        title = _element(record, "title")
        if title is not None:
            bibjson.title = title

        # authors
        ## first we need to extract the affiliations
        affiliations = {}
        affel = record.find("affiliationsList")
        if affel is not None:
            for ael in affel:
                affid = ael.get("affiliationId")
                aff = ael.text
                affiliations[affid] = aff
        ## now crosswalk each author and dereference their affiliation from the table
        authorsel = record.find("authors")
        if authorsel is not None:
            for ael in authorsel:
                name = _element(ael, "name")
                affid = _element(ael, "affiliationId")
                aff = affiliations.get(affid)
                bibjson.add_author(name, affiliation=aff)

        # abstract
        abstract = _element(record, "abstract")
        if abstract is not None:
            bibjson.abstract = abstract[:30000]  # avoids Elasticsearch
            # exceptions about .exact analyser not being able to handle
            # more than 32766 UTF8 characters

        # fulltext
        ftel = record.find("fullTextUrl")
        if ftel is not None and ftel.text is not None and ftel.text != "":
            ct = ftel.get("format")
            url = ftel.text

            try:
                # url validation
                result = urlparse(ftel.text)
                if all([result.scheme, result.netloc, result.path]):
                    bibjson.add_url(url, "fulltext", ct)
                else:
                    raise CrosswalkException(message="Invalid URL: {}".format(ftel.text), inner=e)
            except Exception as e:
                raise CrosswalkException(message="Invalid URL: {}".format(ftel.text), inner=e)

        # keywords
        keyel = record.find("keywords")
        if keyel is not None:
            for kel in keyel:
                if kel.text != "":
                    bibjson.add_keyword(kel.text)

        # add the journal info if requested
        if add_journal_info:
            article.add_journal_metadata()

        return article

###############################################################################
## some convenient utilities
###############################################################################

def _year_month(date):
    try:
        stamp = datetime.strptime(date, "%Y-%m-%d")
        return stamp.year, stamp.month
    except:
        pass
    try:
        stamp = datetime.strptime(date, "%Y-%m")
        return stamp.year, stamp.month
    except:
        pass
    try:
        stamp = datetime.strptime(date, "%Y")
        return stamp.year, None
    except:
        pass
    return None, None

def _element(xml, field):
    el = xml.find(field)
    if el is not None and el.text is not None and el.text != "":
        return el.text.strip()
    return None<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Default configuration for the application

This data must be supplemented with custom configuration to which
URL_SHORTENER_CONFIGURATION environment variable points, overriding
some of the values specified here.

:var SQLALCHEMY_DATABASE_URI: uri of database to be used by the application.

The default value serves only as documentation, and it was taken from:
http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls

:var MIN_NEW_ALIAS_LENGTH: a minimum number of characters in a newly
generated alias
:var MAX_NEW_ALIAS_LENGTH: a maximum number of characters in a newly
generated alias
:var SECRET_KEY: a secret key to be used by the application
:var LOG_FILE: a name of file to which the application writes logs.
The value can be None, in which case the application won't write logs
to any file.

If the value is set, it must consist of an existing directory (if any),
but the file doesn't have to exist. The application uses timed roating
file handler with interval of one day, so the file specified here will
be created (if it doesn't exist yet) and used for one day. After that,
it will be renamed, and another file with the same name will be created
in the same directory and used.

:var INTEGRITY_ERROR_LIMIT: a maximum number of integrity errors allowed to
occur when handling a request for a shortened URL, before logging a warning

:var GOOGLE_SAFE_BROWSING_API_KEY: a value necessary for querying
Google Safe Browsing API.
:var HOST_BLACKLIST_FILE: a name of file storing blacklisted hosts
(hostnames and ip addresses). The value must be either None, or
a name of an existing file containing blacklisted hosts in
a sorted order.

If the value is None, no host blacklist file will be used.

:var RECAPTCHA_PUBLIC_KEY: a value used as a public key for reCAPTCHA,
provided by Google: https://developers.google.com/recaptcha/docs/start

:var RECAPTCHA_PRIVATE_KEY: a value used as a private key for reCAPTCHA,
provided by Google: https://developers.google.com/recaptcha/docs/start

:var ADMIN_EMAIL: email address of administrator(s) of the service provided
by an installation of this application
"""
SQLALCHEMY_DATABASE_URI = (
    'dialect+driver://username:password@host:port/database'
)
MIN_NEW_ALIAS_LENGTH = 1
MAX_NEW_ALIAS_LENGTH = 4
SECRET_KEY = 'a secret key'
LOG_FILE = None
INTEGRITY_ERROR_LIMIT = 10
GOOGLE_SAFE_BROWSING_API_KEY = 'a key'
HOST_BLACKLIST_FILE = None
RECAPTCHA_PUBLIC_KEY = 'public-recaptcha-key'
RECAPTCHA_PRIVATE_KEY = 'private-recaptcha-key'
ADMIN_EMAIL = 'admin@your-domain.com'
<commit_msg>Add BLACKLISTED_HOSTS configuration option<commit_after># -*- coding: utf-8 -*-
""" Default configuration for the application

This data must be supplemented with custom configuration to which
URL_SHORTENER_CONFIGURATION environment variable points, overriding
some of the values specified here.

:var SQLALCHEMY_DATABASE_URI: uri of database to be used by the application.

The default value serves only as documentation, and it was taken from:
http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls

:var MIN_NEW_ALIAS_LENGTH: a minimum number of characters in a newly
generated alias
:var MAX_NEW_ALIAS_LENGTH: a maximum number of characters in a newly
generated alias
:var SECRET_KEY: a secret key to be used by the application
:var LOG_FILE: a name of file to which the application writes logs.
The value can be None, in which case the application won't write logs
to any file.

If the value is set, it must consist of an existing directory (if any),
but the file doesn't have to exist. The application uses timed roating
file handler with interval of one day, so the file specified here will
be created (if it doesn't exist yet) and used for one day. After that,
it will be renamed, and another file with the same name will be created
in the same directory and used.

:var INTEGRITY_ERROR_LIMIT: a maximum number of integrity errors allowed to
occur when handling a request for a shortened URL, before logging a warning

:var GOOGLE_SAFE_BROWSING_API_KEY: a value necessary for querying
Google Safe Browsing API.
:var HOST_BLACKLIST_FILE: a name of file storing blacklisted hosts
(hostnames and ip addresses). The value must be either None, or
a name of an existing file containing blacklisted hosts in
a sorted order.

If the value is None, no host blacklist file will be used.

:var RECAPTCHA_PUBLIC_KEY: a value used as a public key for reCAPTCHA,
provided by Google: https://developers.google.com/recaptcha/docs/start

:var RECAPTCHA_PRIVATE_KEY: a value used as a private key for reCAPTCHA,
provided by Google: https://developers.google.com/recaptcha/docs/start

:var ADMIN_EMAIL: email address of administrator(s) of the service provided
by an installation of this application

:var BLACKLISTED_HOSTS: a custom list of strings representing blacklisted
hosts
"""
SQLALCHEMY_DATABASE_URI = (
    'dialect+driver://username:password@host:port/database'
)
MIN_NEW_ALIAS_LENGTH = 1
MAX_NEW_ALIAS_LENGTH = 4
SECRET_KEY = 'a secret key'
LOG_FILE = None
INTEGRITY_ERROR_LIMIT = 10
GOOGLE_SAFE_BROWSING_API_KEY = 'a key'
HOST_BLACKLIST_FILE = None
RECAPTCHA_PUBLIC_KEY = 'public-recaptcha-key'
RECAPTCHA_PRIVATE_KEY = 'private-recaptcha-key'
ADMIN_EMAIL = 'admin@your-domain.com'
BLACKLISTED_HOSTS = []
<|endoftext|>


<commit_before>#!/usr/bin/python
import urllib.request, urllib.error, urllib.parse
import json
import datetime

# write to JSON file and properly formats the file
def write_to_file(nameOfFile, data):
    with open(nameOfFile, 'w+') as f:
        print("Dumping JSON to", nameOfFile)
        json.dump(data, f, sort_keys=True, indent=4, separators=(',', ':'))

# get the data using StockTwits API
def get_twits(ticker):
    url = "https://api.stocktwits.com/api/2/streams/symbol/{0}.json".format(ticker)
    connection = urllib.request.urlopen(url)
    data = connection.read()
    connection.close()
    return json.loads(data)

# loops through to get data for each ticker in the tickers list
def get_twits_list(tickers):
    ret = {}
    for ticker in tickers:
        print("Getting data for", ticker)
        # error handling
        try:
            data = get_twits(ticker)
            symbol = data['symbol']['symbol']
            msgs = data['messages']
            ret.update({symbol: msgs})
        except Exception as e:
            print(e)
            print("Error getting", ticker)
    return ret

def read_tickers():
    print("Reading tickers from \"tickers.txt\":")
    f = open("tickers.txt", 'r')
    names = []
    # read tickers from tickers.txt
    for line in f:
    	line = line.strip('\n')
    	line = line.upper()
    	line = line.strip('\t')
    	names.append(line)
    print(names)
    return names

def remove_old(original, age_limit=30):
    print("Removing tweets that are more than", age_limit, "days old")
    threshold = datetime.datetime.now() - datetime.timedelta(age_limit)
    result = {}
    # checks if data is more than age_limit and removes if so
    for ticker in list(original.keys()):
        result[ticker] = []
        for msg in original[ticker]:
            dt = datetime.datetime.strptime(msg["created_at"], "%Y-%m-%dT%H:%M:%SZ")
            if dt >= threshold:
                result[ticker].append(msg)
    return result

# default file name
FILENAME = "stocktwits.json"

if __name__ == "__main__":

    # Optional file output
    print("Do you want to specify name of file?")
    x = input()

    # Execute this code if option is taken
    if x.startswith("y") or x.startswith("Y"):
        print("What is file name?")
        filename = input()
        if not (filename.endswith(".json")):
            filename = filename + ".json"
        FILENAME = filename

    # get list of ticker codes
    codes = read_tickers()
    # for each ticker code get the data
    twitdata = get_twits_list(codes)
    # remove the data if older than X days, useless
    twitdata = remove_old(twitdata, 21)
    # write to json file
    write_to_file(FILENAME, twitdata)
<commit_msg>converting to python3<commit_after>#!/usr/bin/python
import urllib.request, urllib.error, urllib.parse
import json
import datetime

# write to JSON file and properly formats the file
def write_to_file(nameOfFile, data):
    with open(nameOfFile, 'w+') as f:
        print("Dumping JSON to", nameOfFile)
        json.dump(data, f, sort_keys=True, indent=4, separators=(',', ':'))

# get the data using StockTwits API
def get_twits(ticker):
    url = "https://api.stocktwits.com/api/2/streams/symbol/{0}.json".format(ticker)
    connection = urllib.request.urlopen(url)
    data = connection.read()
    connection.close()
    return json.loads(data)

# loops through to get data for each ticker in the tickers list
def get_twits_list(tickers):
    ret = {}
    for ticker in tickers:
        print("Getting data for", ticker)
        # error handling
        try:
            data = get_twits(ticker)
            symbol = data['symbol']['symbol']
            msgs = data['messages']
            ret.update({symbol: msgs})
        except Exception as e:
            print(e)
            print("Error getting", ticker)
    return ret

def read_tickers():
    print("Reading tickers from \"tickers.txt\":")
    f = open("tickers.txt", 'r')
    names = []
    # read tickers from tickers.txt
    for line in f:
    	line = line.strip('\n')
    	line = line.upper()
    	line = line.strip('\t')
    	names.append(line)
    print(names)
    return names

def remove_old(original, age_limit=30):
    print("Removing tweets that are more than", age_limit, "days old")
    threshold = datetime.datetime.now() - datetime.timedelta(age_limit)
    result = {}
    # checks if data is more than age_limit and removes if so
    for ticker in list(original.keys()):
        result[ticker] = []
        for msg in original[ticker]:
            dt = datetime.datetime.strptime(msg["created_at"], "%Y-%m-%dT%H:%M:%SZ")
            if dt >= threshold:
                result[ticker].append(msg)
    return result

# default file name
FILENAME = "stocktwits.json"

if __name__ == "__main__":

    # Optional file output
    print("Do you want to specify name of output file?")
    x = input()

    # Execute this code if option is taken
    if x.startswith("y") or x.startswith("Y"):
        print("What is file name?")
        filename = input()
        if not (filename.endswith(".json")):
            filename = filename + ".json"
        FILENAME = filename

    # get list of ticker codes
    codes = read_tickers()
    # for each ticker code get the data
    twitdata = get_twits_list(codes)
    # remove the data if older than X days, useless
    twitdata = remove_old(twitdata, 21)
    # write to json file
    write_to_file(FILENAME, twitdata)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

#########################################################################
#########################################################################

"""
   File Name: indexer.py
      Author: Wan Ji
      E-mail: wanji@live.com
  Created on: Tue Nov  4 09:07:38 2014 CST
"""
DESCRIPTION = """
"""

import cPickle as pickle
import logging

import numpy as np

from util import kmeans, pq_kmeans_assign, pq_knn
from distance import distFunc
from storage import createStorage

import _cext as cext

class Indexer(object):
    class IdxData(object):
        pass

    def __init__(self):
        self.ERR_INSTAN = "Instance of `Indexer` is not allowed!"
        self.ERR_UNIMPL = "Unimplemented method!"
        pass

    def __del__(self):
        pass

    def build(self, vals=None, labels=None):
        """
        Build the indexer based on given training data
        """
        raise Exception(self.ERR_INSTAN)

    def load(self, path):
        """
        Load indexer information from file
        """
        with open(path, 'rb') as pklf:
            self.idxdat = pickle.load(pklf)

    def save(self, path):
        """
        Save the information related to the indexer itself
        """
        with open(path, 'wb') as pklf:
            pickle.dump(self.idxdat, pklf, protocol=2)

    def set_storage(self, storage_type='mem', storage_parm=None):
        """
        Set up the backend storage engine
        """
        raise Exception(self.ERR_INSTAN)

    def add(self, vals, keys):
        """
        Add one or more items to the indexer
        """
        raise Exception(self.ERR_INSTAN)

    def remove(self, keys):
        """
        Remove one or more items from the indexer
        """
        raise Exception(self.ERR_INSTAN)

    def search(self, querys, topk=None, thresh=None):
        """
        Search in the indexer for `k` nearest neighbors or
        neighbors in a distance of `thresh`
        """
        raise Exception(self.ERR_INSTAN)

class PQIndexer(Indexer):
    def __init__(self):
        Indexer.__init__(self)

    def __del__(self):
        pass

    def build(self, pardic=None):
        # training data
        vals = pardic['vals']
        # the number of subquantizers
        nsubq = pardic['nsubq']
        # the number bits of each subquantizer
        nsubqbits = pardic.get('nsubqbits', 8)
        # the number of items in one block

        blksize = pardic.get('blksize', 16384)

        # vector dimension
        dim = vals.shape[1]
        # dimension of the subvectors to quantize
        dsub = dim / nsubq
        # number of centroids per subquantizer
        ksub = 2 ** nsubqbits

        """
        Initializing indexer data
        """
        idxdat = dict()
        idxdat['nsubq'] = nsubq
        idxdat['ksub'] = ksub
        idxdat['dsub'] = dsub
        idxdat['blksize'] = blksize
        idxdat['centroids'] = [None for q in range(nsubq)]

        logging.info("Building codebooks in subspaces - BEGIN")
        for q in range(nsubq):
            logging.info("\tsubspace %d/%d:" % (q, nsubq))
            vs = np.require(vals[:, q*dsub:(q+1)*dsub], requirements='C')
            idxdat['centroids'][q] = kmeans(vs, ksub, niter=100)
        logging.info("Building codebooks in subspaces - DONE")

        self.idxdat = idxdat

    def set_storage(self, storage_type='mem', storage_parm=None):
        self.storage = createStorage(storage_type, storage_parm)

    def add(self, vals, keys=None):
        num_vals = vals.shape[0]
        if keys is None:
            num_base_items = self.storage.get_num_items()
            keys = range(num_base_items, num_base_items + num_vals)

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        centroids = self.idxdat['centroids']

        blksize = self.idxdat.get('blksize', 16384)
        start_id = 0
        for start_id in range(0, num_vals, blksize):
            cur_num = min(blksize, num_vals - start_id)
            print "%8d/%d: %d" % (start_id, num_vals, cur_num)

            codes = np.zeros((cur_num, nsubq), np.uint8)
            for q in range(nsubq):
                vsub = vals[start_id:start_id+cur_num, q*dsub:(q+1)*dsub]
                codes[:, q] = pq_kmeans_assign(centroids[q], vsub)
            self.storage.add(codes, keys[start_id:start_id+cur_num])

    def remove(self, keys):
        raise Exception(self.ERR_UNIMPL)

    def search(self, querys, topk=None, thresh=None):
        nq = querys.shape[0]

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        ksub = self.idxdat['ksub']
        centroids = self.idxdat['centroids']

        distab = np.zeros((nsubq, ksub), np.single)
        dis = np.zeros((nq, topk), np.single)
        ids = np.zeros((nq, topk), np.int)

        for qid in range(nq):
            # pre-compute the table of squared distance to centroids
            for q in range(nsubq):
                vsub = querys[qid:qid+1, q*dsub:(q+1)*dsub]
                distab[q:q+1, :] = distFunc['euclidean'](
                    centroids[q], vsub)

            # add the tabulated distances to construct the distance estimators
            idsquerybase, disquerybase = self.sumidxtab(distab)
            cur_ids = pq_knn(disquerybase, topk)

            ids[qid, :] = idsquerybase[cur_ids]
            dis[qid, :] = disquerybase[cur_ids]

        return ids, dis

    def sumidxtab(self, D):
        """
        Compute distance to database items based on distances to centroids.
            D: nsubq x ksub
        """

        num_base_items = self.storage.get_num_items()

        dis = np.zeros(num_base_items)
        ids = np.arange(0)

        start_id = 0
        for keys, blk in self.storage:
            cur_num = blk.shape[0]
            # dis[start_id:start_id+cur_num] = self.sumidxtab_core(D, blk)
            dis[start_id:start_id+cur_num] = cext.sumidxtab_core(D, blk)
            start_id += cur_num
            ids = np.hstack((ids, keys))

        return ids, dis

    @classmethod
    def sumidxtab_core(cls, D, blk):
        # return 0
        return [sum([D[j, blk[i, j]] for j in range(D.shape[0])])
                for i in range(blk.shape[0])]

class IVFPQIndexer(PQIndexer):
    def __init__(self):
        PQIndexer.__init__(self)

    def __del__(self):
        pass

    def build(self, pardic=None):
        # training data
        vals = pardic['vals']
        # the number of coarse centroids
        coarsek = pardic['coarsek']

        logging.info('Building coarse quantizer - BEGIN')
        coa_centroids = kmeans(vals, coarsek, niter=100)
        cids = pq_kmeans_assign(coa_centroids, vals)
        logging.info('Building coarse quantizer - DONE')

        pardic['vals'] -= coa_centroids[cids, :]

        PQIndexer.build(self, pardic)

        self.idxdat['coa_centroids'] = coa_centroids
        self.idxdat['coarsek'] = coarsek

    def set_storage(self, storage_type='mem', storage_parm=None):
        storage_parm['coarsek'] = self.idxdat['coarsek']
        self.storage = createStorage(storage_type, storage_parm)

    def add(self, vals, keys=None):
        num_vals = vals.shape[0]
        if keys is None:
            num_base_items = sum([ivf.get_num_items() for ivf in self.storage])
            keys = np.arange(num_base_items, num_base_items + num_vals)

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        centroids = self.idxdat['centroids']
        coarsek = self.idxdat['coarsek']
        coa_centroids = self.idxdat['coa_centroids']

        blksize = self.idxdat.get('blksize', 16384)
        start_id = 0
        for start_id in range(0, num_vals, blksize):
            cur_num = min(blksize, num_vals - start_id)
            end_id = start_id + cur_num
            print "%8d/%d: %d" % (start_id, num_vals, cur_num)

            cur_vals = vals[start_id:end_id, :]
            cur_cids = pq_kmeans_assign(coa_centroids, cur_vals)
            cur_vals -= coa_centroids[cur_cids, :]

            codes = np.zeros((cur_num, nsubq), np.uint8)
            for q in range(nsubq):
                vsub = cur_vals[:, q*dsub:(q+1)*dsub]
                codes[:, q] = pq_kmeans_assign(centroids[q], vsub)
            # self.storage.add(codes, keys)
            for ivfidx in xrange(coarsek):
                self.storage[ivfidx].add(
                    codes[cur_cids == ivfidx, :],
                    keys[start_id:end_id][cur_cids == ivfidx])

    def remove(self, keys):
        raise Exception(self.ERR_UNIMPL)

    def search(self, querys, topk=None, thresh=None, coa_nn=8):
        nq = querys.shape[0]

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        ksub = self.idxdat['ksub']
        centroids = self.idxdat['centroids']
        coa_centroids = self.idxdat['coa_centroids']

        distab = np.zeros((nsubq, ksub), np.single)
        dis = np.zeros((nq, topk), np.single)
        ids = np.zeros((nq, topk), np.int)

        coa_dist = distFunc['euclidean'](coa_centroids, querys)
        for qid in range(nq):
            query = querys[qid:qid+1, :]
            coa_knn = pq_knn(coa_dist[qid, :], coa_nn)
            query -= coa_centroids[coa_knn[0], :]
            # pre-compute the table of squared distance to centroids
            for q in range(nsubq):
                vsub = query[:, q*dsub:(q+1)*dsub]
                distab[q:q+1, :] = distFunc['euclidean'](
                    centroids[q], vsub)

            # add the tabulated distances to construct the distance estimators
            idsquerybase, disquerybase = self.sumidxtab(
                distab, coa_knn)
            cur_ids = pq_knn(disquerybase, topk)

            ids[qid, :] = idsquerybase[cur_ids]
            dis[qid, :] = disquerybase[cur_ids]

        return ids, dis

    def sumidxtab(self, D, coa_knn):
        """
        Compute distance to database items based on distances to centroids.
            D: nsubq x ksub
        """

        num_candidates = sum([self.storage[ivfidx].get_num_items()
                              for ivfidx in coa_knn])
        dis = np.zeros(num_candidates)
        ids = np.arange(0)

        start_id = 0

        for ivfidx in coa_knn:
            for keys, blk in self.storage[ivfidx]:
                cur_num = blk.shape[0]
                # dis[start_id:start_id+cur_num] = self.sumidxtab_core(D, blk)
                dis[start_id:start_id+cur_num] = cext.sumidxtab_core(D, blk)
                start_id += cur_num
                ids = np.hstack((ids, keys))

        return ids, dis
<commit_msg>fix bug: prevent the ivfpq-indexer from modifying the parameters<commit_after>#!/usr/bin/env python
# coding: utf-8

#########################################################################
#########################################################################

"""
   File Name: indexer.py
      Author: Wan Ji
      E-mail: wanji@live.com
  Created on: Tue Nov  4 09:07:38 2014 CST
"""
DESCRIPTION = """
"""

import cPickle as pickle
import logging

import numpy as np

from util import kmeans, pq_kmeans_assign, pq_knn
from distance import distFunc
from storage import createStorage

import _cext as cext

class Indexer(object):
    class IdxData(object):
        pass

    def __init__(self):
        self.ERR_INSTAN = "Instance of `Indexer` is not allowed!"
        self.ERR_UNIMPL = "Unimplemented method!"
        pass

    def __del__(self):
        pass

    def build(self, vals=None, labels=None):
        """
        Build the indexer based on given training data
        """
        raise Exception(self.ERR_INSTAN)

    def load(self, path):
        """
        Load indexer information from file
        """
        with open(path, 'rb') as pklf:
            self.idxdat = pickle.load(pklf)

    def save(self, path):
        """
        Save the information related to the indexer itself
        """
        with open(path, 'wb') as pklf:
            pickle.dump(self.idxdat, pklf, protocol=2)

    def set_storage(self, storage_type='mem', storage_parm=None):
        """
        Set up the backend storage engine
        """
        raise Exception(self.ERR_INSTAN)

    def add(self, vals, keys):
        """
        Add one or more items to the indexer
        """
        raise Exception(self.ERR_INSTAN)

    def remove(self, keys):
        """
        Remove one or more items from the indexer
        """
        raise Exception(self.ERR_INSTAN)

    def search(self, queries, topk=None, thresh=None):
        """
        Search in the indexer for `k` nearest neighbors or
        neighbors in a distance of `thresh`
        """
        raise Exception(self.ERR_INSTAN)

class PQIndexer(Indexer):
    def __init__(self):
        Indexer.__init__(self)

    def __del__(self):
        pass

    def build(self, pardic=None):
        # training data
        vals = pardic['vals']
        # the number of subquantizers
        nsubq = pardic['nsubq']
        # the number bits of each subquantizer
        nsubqbits = pardic.get('nsubqbits', 8)
        # the number of items in one block

        blksize = pardic.get('blksize', 16384)

        # vector dimension
        dim = vals.shape[1]
        # dimension of the subvectors to quantize
        dsub = dim / nsubq
        # number of centroids per subquantizer
        ksub = 2 ** nsubqbits

        """
        Initializing indexer data
        """
        idxdat = dict()
        idxdat['nsubq'] = nsubq
        idxdat['ksub'] = ksub
        idxdat['dsub'] = dsub
        idxdat['blksize'] = blksize
        idxdat['centroids'] = [None for q in range(nsubq)]

        logging.info("Building codebooks in subspaces - BEGIN")
        for q in range(nsubq):
            logging.info("\tsubspace %d/%d:" % (q, nsubq))
            vs = np.require(vals[:, q*dsub:(q+1)*dsub], requirements='C')
            idxdat['centroids'][q] = kmeans(vs, ksub, niter=100)
        logging.info("Building codebooks in subspaces - DONE")

        self.idxdat = idxdat

    def set_storage(self, storage_type='mem', storage_parm=None):
        self.storage = createStorage(storage_type, storage_parm)

    def add(self, vals, keys=None):
        num_vals = vals.shape[0]
        if keys is None:
            num_base_items = self.storage.get_num_items()
            keys = range(num_base_items, num_base_items + num_vals)

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        centroids = self.idxdat['centroids']

        blksize = self.idxdat.get('blksize', 16384)
        start_id = 0
        for start_id in range(0, num_vals, blksize):
            cur_num = min(blksize, num_vals - start_id)
            print "%8d/%d: %d" % (start_id, num_vals, cur_num)

            codes = np.zeros((cur_num, nsubq), np.uint8)
            for q in range(nsubq):
                vsub = vals[start_id:start_id+cur_num, q*dsub:(q+1)*dsub]
                codes[:, q] = pq_kmeans_assign(centroids[q], vsub)
            self.storage.add(codes, keys[start_id:start_id+cur_num])

    def remove(self, keys):
        raise Exception(self.ERR_UNIMPL)

    def search(self, queries, topk=None, thresh=None):
        nq = queries.shape[0]

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        ksub = self.idxdat['ksub']
        centroids = self.idxdat['centroids']

        distab = np.zeros((nsubq, ksub), np.single)
        dis = np.zeros((nq, topk), np.single)
        ids = np.zeros((nq, topk), np.int)

        for qid in range(nq):
            # pre-compute the table of squared distance to centroids
            for q in range(nsubq):
                vsub = queries[qid:qid+1, q*dsub:(q+1)*dsub]
                distab[q:q+1, :] = distFunc['euclidean'](
                    centroids[q], vsub)

            # add the tabulated distances to construct the distance estimators
            idsquerybase, disquerybase = self.sumidxtab(distab)
            cur_ids = pq_knn(disquerybase, topk)

            ids[qid, :] = idsquerybase[cur_ids]
            dis[qid, :] = disquerybase[cur_ids]

        return ids, dis

    def sumidxtab(self, D):
        """
        Compute distance to database items based on distances to centroids.
            D: nsubq x ksub
        """

        num_base_items = self.storage.get_num_items()

        dis = np.zeros(num_base_items)
        ids = np.arange(0)

        start_id = 0
        for keys, blk in self.storage:
            cur_num = blk.shape[0]
            # dis[start_id:start_id+cur_num] = self.sumidxtab_core(D, blk)
            dis[start_id:start_id+cur_num] = cext.sumidxtab_core(D, blk)
            start_id += cur_num
            ids = np.hstack((ids, keys))

        return ids, dis

    @classmethod
    def sumidxtab_core(cls, D, blk):
        # return 0
        return [sum([D[j, blk[i, j]] for j in range(D.shape[0])])
                for i in range(blk.shape[0])]

class IVFPQIndexer(PQIndexer):
    def __init__(self):
        PQIndexer.__init__(self)

    def __del__(self):
        pass

    def build(self, pardic=None):
        # training data
        vals = pardic['vals']
        # the number of coarse centroids
        coarsek = pardic['coarsek']

        logging.info('Building coarse quantizer - BEGIN')
        coa_centroids = kmeans(vals, coarsek, niter=100)
        cids = pq_kmeans_assign(coa_centroids, vals)
        logging.info('Building coarse quantizer - DONE')

        pardic['vals'] -= coa_centroids[cids, :]

        PQIndexer.build(self, pardic)

        self.idxdat['coa_centroids'] = coa_centroids
        self.idxdat['coarsek'] = coarsek

    def set_storage(self, storage_type='mem', storage_parm=None):
        storage_parm['coarsek'] = self.idxdat['coarsek']
        self.storage = createStorage(storage_type, storage_parm)

    def add(self, vals, keys=None):
        num_vals = vals.shape[0]
        if keys is None:
            num_base_items = sum([ivf.get_num_items() for ivf in self.storage])
            keys = np.arange(num_base_items, num_base_items + num_vals)

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        centroids = self.idxdat['centroids']
        coarsek = self.idxdat['coarsek']
        coa_centroids = self.idxdat['coa_centroids']

        blksize = self.idxdat.get('blksize', 16384)
        start_id = 0
        for start_id in range(0, num_vals, blksize):
            cur_num = min(blksize, num_vals - start_id)
            end_id = start_id + cur_num
            print "%8d/%d: %d" % (start_id, num_vals, cur_num)

            # Here `copy()` is necessary, it ensures that you DONOT modify the vals
            cur_vals = vals[start_id:end_id, :].copy()
            cur_cids = pq_kmeans_assign(coa_centroids, cur_vals)
            cur_vals -= coa_centroids[cur_cids, :]

            codes = np.zeros((cur_num, nsubq), np.uint8)
            for q in range(nsubq):
                vsub = cur_vals[:, q*dsub:(q+1)*dsub]
                codes[:, q] = pq_kmeans_assign(centroids[q], vsub)
            # self.storage.add(codes, keys)
            for ivfidx in xrange(coarsek):
                self.storage[ivfidx].add(
                    codes[cur_cids == ivfidx, :],
                    keys[start_id:end_id][cur_cids == ivfidx])

    def remove(self, keys):
        raise Exception(self.ERR_UNIMPL)

    def search(self, queries, topk=None, thresh=None, coa_nn=8):
        nq = queries.shape[0]

        dsub = self.idxdat['dsub']
        nsubq = self.idxdat['nsubq']
        ksub = self.idxdat['ksub']
        centroids = self.idxdat['centroids']
        coa_centroids = self.idxdat['coa_centroids']

        distab = np.zeros((nsubq, ksub), np.single)
        dis = np.zeros((nq, topk), np.single)
        ids = np.zeros((nq, topk), np.int)

        coa_dist = distFunc['euclidean'](coa_centroids, queries)
        for qid in range(nq):
            # Here `copy()` is necessary, it ensures that you DONOT modify the queries
            query = queries[qid:qid+1, :].copy()
            coa_knn = pq_knn(coa_dist[qid, :], coa_nn)
            query -= coa_centroids[coa_knn[0], :]
            # pre-compute the table of squared distance to centroids
            for q in range(nsubq):
                vsub = query[:, q*dsub:(q+1)*dsub]
                distab[q:q+1, :] = distFunc['euclidean'](
                    centroids[q], vsub)

            # add the tabulated distances to construct the distance estimators
            idsquerybase, disquerybase = self.sumidxtab(
                distab, coa_knn)
            cur_ids = pq_knn(disquerybase, topk)

            ids[qid, :] = idsquerybase[cur_ids]
            dis[qid, :] = disquerybase[cur_ids]

        return ids, dis

    def sumidxtab(self, D, coa_knn):
        """
        Compute distance to database items based on distances to centroids.
            D: nsubq x ksub
        """

        num_candidates = sum([self.storage[ivfidx].get_num_items()
                              for ivfidx in coa_knn])
        dis = np.zeros(num_candidates)
        ids = np.arange(0)

        start_id = 0

        for ivfidx in coa_knn:
            for keys, blk in self.storage[ivfidx]:
                cur_num = blk.shape[0]
                # dis[start_id:start_id+cur_num] = self.sumidxtab_core(D, blk)
                dis[start_id:start_id+cur_num] = cext.sumidxtab_core(D, blk)
                start_id += cur_num
                ids = np.hstack((ids, keys))

        return ids, dis
<|endoftext|>


<commit_before>import sys
import time
import logging
import numpy as np

if len(sys.argv) == 1:
    print("ERROR: Please specify implementation to benchmark, 'sknn' 'dbn' or 'lasagne'.")
    sys.exit(-1)

np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)

log = logging.getLogger()
log.setLevel(logging.DEBUG)

stdout = logging.StreamHandler(sys.stdout)
stdout.setLevel(logging.DEBUG)
stdout.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
log.addHandler(stdout)

from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_mldata

mnist = fetch_mldata('MNIST original')
X_train, X_test, y_train, y_test = train_test_split(
        (mnist.data / 255.0).astype(np.float32),
        mnist.target.astype(np.int32),
        test_size=0.33, random_state=1234)

classifiers = []

if 'dbn' in sys.argv:
    from nolearn.dbn import DBN
    clf = DBN(
        [X_train.shape[1], 300, 10],
        learn_rates=0.3,
        learn_rate_decays=0.9,
        epochs=10,
        verbose=1)
    classifiers.append(('nolearn.dbn', clf))

if 'sknn' in sys.argv:
    from sknn.mlp import MultiLayerPerceptronClassifier

    clf = MultiLayerPerceptronClassifier(
        layers=[("Rectifier", 300), ("Softmax",)],
        learning_rate=0.02,
        learning_rule='momentum',
        batch_size=25,
        n_stable=10,
        n_iter=10,
        verbose=0,
    )
    classifiers.append(('sknn.mlp', clf))

if 'lasagne' in sys.argv:
    from nolearn.lasagne import NeuralNet
    from lasagne.layers import InputLayer, DenseLayer
    from lasagne.nonlinearities import softmax
    from lasagne.updates import nesterov_momentum

    clf = NeuralNet(
        layers=[
            ('input', InputLayer),
            ('hidden1', DenseLayer),
            ('output', DenseLayer),
            ],
        input_shape=(None, 784),
        output_num_units=10,
        output_nonlinearity=softmax,
        eval_size=0.0,

        more_params=dict(
            hidden1_num_units=300,
        ),

        update=nesterov_momentum,
        update_learning_rate=0.02,
        update_momentum=0.9,

        max_epochs=10,
        verbose=1
        )
    classifiers.append(('nolearn.lasagne', clf))

for name, clf in classifiers:
    start = time.time()
    clf.fit(X_train, y_train)

    from sklearn.metrics import classification_report

    y_pred = clf.predict(X_test)
    print name
    print "\tAccuracy:", clf.score(X_test, y_test)
    print "\tTime:", time.time() - start
    print "\tReport:"
    print classification_report(y_test, y_pred)
<commit_msg>Using the official slug from mldata.org to try to prevent problems on some machines?<commit_after>import sys
import time
import logging
import numpy as np

if len(sys.argv) == 1:
    print("ERROR: Please specify implementation to benchmark, 'sknn' 'dbn' or 'lasagne'.")
    sys.exit(-1)

np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)

log = logging.getLogger()
log.setLevel(logging.DEBUG)

stdout = logging.StreamHandler(sys.stdout)
stdout.setLevel(logging.DEBUG)
stdout.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
log.addHandler(stdout)

from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_mldata

mnist = fetch_mldata('mnist-original')
X_train, X_test, y_train, y_test = train_test_split(
        (mnist.data / 255.0).astype(np.float32),
        mnist.target.astype(np.int32),
        test_size=0.33, random_state=1234)

classifiers = []

if 'dbn' in sys.argv:
    from nolearn.dbn import DBN
    clf = DBN(
        [X_train.shape[1], 300, 10],
        learn_rates=0.3,
        learn_rate_decays=0.9,
        epochs=10,
        verbose=1)
    classifiers.append(('nolearn.dbn', clf))

if 'sknn' in sys.argv:
    from sknn.mlp import MultiLayerPerceptronClassifier

    clf = MultiLayerPerceptronClassifier(
        layers=[("Rectifier", 300), ("Softmax",)],
        learning_rate=0.02,
        learning_rule='momentum',
        batch_size=25,
        n_stable=10,
        n_iter=10,
        verbose=0,
    )
    classifiers.append(('sknn.mlp', clf))

if 'lasagne' in sys.argv:
    from nolearn.lasagne import NeuralNet
    from lasagne.layers import InputLayer, DenseLayer
    from lasagne.nonlinearities import softmax
    from lasagne.updates import nesterov_momentum

    clf = NeuralNet(
        layers=[
            ('input', InputLayer),
            ('hidden1', DenseLayer),
            ('output', DenseLayer),
            ],
        input_shape=(None, 784),
        output_num_units=10,
        output_nonlinearity=softmax,
        eval_size=0.0,

        more_params=dict(
            hidden1_num_units=300,
        ),

        update=nesterov_momentum,
        update_learning_rate=0.02,
        update_momentum=0.9,

        max_epochs=10,
        verbose=1
        )
    classifiers.append(('nolearn.lasagne', clf))

for name, clf in classifiers:
    start = time.time()
    clf.fit(X_train, y_train)

    from sklearn.metrics import classification_report

    y_pred = clf.predict(X_test)
    print name
    print "\tAccuracy:", clf.score(X_test, y_test)
    print "\tTime:", time.time() - start
    print "\tReport:"
    print classification_report(y_test, y_pred)
<|endoftext|>