<commit_before>""" Set up raw data for the release test.

Copies a set of run folders and selected FASTQ files from the main raw data
folder to your developer workstation. Configure the set of samples to copy
in test_samples.csv.
You'll need to upload the QC data from the InterOp folders before you can
start MISEQ_MONITOR.py.
"""

from argparse import ArgumentParser
from csv import DictReader
from datetime import datetime
import errno
from glob import glob
from itertools import groupby
from operator import attrgetter
import os
from pathlib import Path
from shutil import copy, rmtree

from micall.utils.sample_sheet_parser import sample_sheet_parser

NEEDS_PROCESSING = 'needsprocessing'
ERROR_PROCESSING = 'errorprocessing'

def parse_args():
    parser = ArgumentParser(description='Prepare sample runs for testing a new release.')
    parser.add_argument(
        'old_folder',
        help='Main RAWDATA folder that will be copied to local RAWDATA.')
    parser.add_argument(
        'test_folder',
        type=Path,
        default=os.environ.get('MICALL_RAW_DATA',
                               Path.home() / "data/RAW_DATA"),
        help='Local RAWDATA folder that will be used to run tests.')
    parser.add_argument(
        '-s',
        '--samples_csv',
        default='test_samples.csv',
        help='CSV file with sample and run names.')
    parser.add_argument(
        '-m',
        '--min_run_name',
        help='Select all later runs, (e.g., "161201"). Overrides samples_csv.')
    parser.add_argument(
        '--pipeline_version',
        default='0-dev',
        help='version suffix for folder names')
    return parser.parse_args()

class Sample(object):
    def __init__(self, run_name, extract_num, config):
        self.run_name = run_name
        self.extract_num = extract_num
        self.config = config
        self.fastq_paths = None

    def find(self, source_folder, qai_run_names=None):
        """ Find matching samples in the source folder.

        Puts all the sample names in self.fastq_paths.
        :param str source_folder: the folder to search for samples that match
            self.extract_num
        :param set qai_run_names: a set to add the new run name to as it would
            be formatted on QAI.
        """
        run_path = os.path.join(source_folder, 'MiSeq', 'runs', self.run_name)
        if os.path.exists(run_path):
            self.run_name = run_path
        else:
            run_parts = str(self.run_name).split('.')
            if len(run_parts) < 2:
                run_parts.append('M01841')
            format_string = '%d-%b-%y' if len(run_parts[0]) == 9 else '%d-%b-%Y'
            run_date = datetime.strptime(run_parts[0], format_string)
            base_run_name = run_date.strftime('%y%m%d') + '_' + run_parts[1]
            pattern = os.path.join(source_folder,
                                   'MiSeq',
                                   'runs',
                                   base_run_name+'*',
                                   NEEDS_PROCESSING)
            matches = glob(pattern)
            if len(matches) != 1:
                raise RuntimeError('Expected one match for {}, but found: {}'.format(
                    pattern,
                    matches))
            self.run_name = os.path.dirname(matches[0])

        pattern = os.path.join(self.run_name,
                               'Data',
                               'Intensities',
                               'BaseCalls',
                               self.extract_num + '*_R1_*')
        matches = glob(pattern)
        if len(matches) == 0:
            raise RuntimeError('No matches found for ' + pattern)
        matches.sort()
        self.fastq_paths = matches
        if qai_run_names is not None:
            sample_sheet_path = os.path.join(self.run_name, 'SampleSheet.csv')
            with open(sample_sheet_path) as f:
                try:
                    sample_sheet = sample_sheet_parser(f)
                except ValueError:
                    print(f'Bad sample sheet for {self.run_name}.')
                else:
                    qai_run_name = sample_sheet['Project Name']
                    qai_run_names.add(qai_run_name)

    def setup_samples(self):
        base_run_name = os.path.basename(self.run_name)
        target_data_path = os.path.join(self.config.test_folder,
                                        'MiSeq',
                                        'runs',
                                        base_run_name,
                                        'Data',
                                        'Intensities',
                                        'BaseCalls')
        suspended_path = os.path.join(target_data_path, 'suspended')
        sample_paths = []
        for fastq1_path in self.fastq_paths:
            fastq2_path = fastq1_path.replace('_R1_', '_R2_')
            for fastq_path in (fastq1_path, fastq2_path):
                base_name = os.path.basename(fastq_path)
                target_path = os.path.join(target_data_path, base_name)
                sample_paths.append(target_path)
                if not os.path.exists(target_path):
                    suspended_fastq_path = os.path.join(suspended_path,
                                                        base_name)
                    if os.path.exists(suspended_fastq_path):
                        os.rename(suspended_fastq_path,
                                  target_path)
                    else:
                        os.symlink(fastq_path, target_path)
        return sample_paths

    def setup_run(self):
        base_run_name = os.path.basename(self.run_name)
        target_run_path = os.path.join(self.config.test_folder,
                                       'MiSeq',
                                       'runs',
                                       base_run_name)
        suspended_run_path = os.path.join(self.config.test_folder,
                                          'MiSeq',
                                          'runs',
                                          'suspended',
                                          base_run_name)
        if os.path.exists(target_run_path):
            pass
        elif os.path.exists(suspended_run_path):
            os.rename(suspended_run_path, target_run_path)
        else:
            base_calls_path = os.path.join(target_run_path,
                                           'Data',
                                           'Intensities',
                                           'BaseCalls')
            os.makedirs(base_calls_path)
            os.symlink(os.path.join(self.run_name, 'InterOp'),
                       os.path.join(target_run_path, 'InterOp'))
            for filename in ('RunInfo.xml',
                             'runParameters.xml',
                             'SampleSheet.csv',
                             'needsprocessing'):
                copy(os.path.join(self.run_name, filename),
                     os.path.join(target_run_path, filename))
        results_path = os.path.join(target_run_path,
                                    'Results',
                                    'version_' + self.config.pipeline_version)
        try:
            rmtree(results_path)
        except OSError as ex:
            if ex.errno != errno.ENOENT:
                raise
            os.makedirs(os.path.dirname(results_path), exist_ok=True)
        try:
            os.remove(os.path.join(target_run_path,
                                   ERROR_PROCESSING))
        except OSError as ex:
            if ex.errno != errno.ENOENT:
                raise
        return target_run_path

def suspend_inactive_runs(active_runs, rawdata_mount):
    active_run_names = set(map(os.path.basename, active_runs))
    local_runs = glob(os.path.join(rawdata_mount, 'MiSeq', 'runs', '*'))
    for run_path in local_runs:
        base_run_name = os.path.basename(run_path)
        if base_run_name not in active_run_names and base_run_name != 'suspended':
            suspended_path = os.path.join(rawdata_mount,
                                          'MiSeq',
                                          'runs',
                                          'suspended',
                                          base_run_name)
            os.rename(run_path, suspended_path)

def suspend_inactive_samples(run_path, active_sample_paths):
    base_calls_path = os.path.join(run_path, 'Data', 'Intensities', 'BaseCalls')
    local_samples = glob(os.path.join(base_calls_path, '*'))
    excluded = list(active_sample_paths)
    suspended_path = os.path.join(base_calls_path, 'suspended')
    excluded.append(suspended_path)
    for sample_path in local_samples:
        if sample_path not in excluded:
            if not os.path.exists(suspended_path):
                os.makedirs(suspended_path)
            os.rename(sample_path,
                      os.path.join(suspended_path, os.path.basename(sample_path)))
    try:
        os.rmdir(suspended_path)
    except OSError as ex:
        if ex.errno not in (errno.ENOTEMPTY, errno.ENOENT):
            raise

def find_run_folders(source_folder, min_run_name):
    pattern = os.path.join(source_folder, 'MiSeq', 'runs', '*')
    matches = glob(pattern)
    folder_names = [os.path.basename(match) for match in matches]
    recent_folders = [folder
                      for folder in folder_names
                      if folder >= min_run_name and folder[0].isnumeric()]
    folders_with_data = [folder
                         for folder in recent_folders
                         if os.path.exists(os.path.join(source_folder,
                                                        'MiSeq',
                                                        'runs',
                                                        folder,
                                                        NEEDS_PROCESSING))]
    return folders_with_data

def main():
    args = parse_args()
    if args.min_run_name is not None:
        print('Setting up',
              args.pipeline_version,
              'for runs after',
              args.min_run_name)
        run_folders = find_run_folders(args.old_folder, args.min_run_name)
        samples = [Sample(folder, '*', args) for folder in run_folders]
    else:
        print('Setting up',
              args.pipeline_version,
              'for runs in',
              args.samples_csv)
        with open(args.samples_csv, 'r') as samples_csv:
            samples = [Sample(row['run'], row['enum'], args)
                       for row in DictReader(samples_csv)]
    qai_run_names = set()
    for sample in samples:
        sample.find(args.old_folder, qai_run_names)
    for qai_run_name in sorted(qai_run_names):
        print('LabMiseqRun.import("{}")'.format(qai_run_name))
    samples.sort(key=lambda s: (s.run_name, s.extract_num))
    runs = []
    for run, run_samples in groupby(samples, key=attrgetter('run_name')):
        runs.append(run)
        run_name = os.path.basename(run)
        print('--' + run_name)
        sample_paths = []
        run_path = None
        for sample in run_samples:
            run_path = sample.setup_run()
            sample_paths.extend(sample.setup_samples())
            print('  ' + ', '.join(map(os.path.basename, sample.fastq_paths)))
        suspend_inactive_samples(run_path, active_sample_paths=sample_paths)
    suspend_inactive_runs(active_runs=runs, rawdata_mount=args.test_folder)

if __name__ == '__main__':
    main()
<commit_msg>Stop copying unused file in release_test_setup.py.<commit_after>""" Set up raw data for the release test.

Copies a set of run folders and selected FASTQ files from the main raw data
folder to your developer workstation. Configure the set of samples to copy
in test_samples.csv.
You'll need to upload the QC data from the InterOp folders before you can
start MISEQ_MONITOR.py.
"""

from argparse import ArgumentParser
from csv import DictReader
from datetime import datetime
import errno
from glob import glob
from itertools import groupby
from operator import attrgetter
import os
from pathlib import Path
from shutil import copy, rmtree

from micall.utils.sample_sheet_parser import sample_sheet_parser

NEEDS_PROCESSING = 'needsprocessing'
ERROR_PROCESSING = 'errorprocessing'

def parse_args():
    parser = ArgumentParser(description='Prepare sample runs for testing a new release.')
    parser.add_argument(
        'old_folder',
        help='Main RAWDATA folder that will be copied to local RAWDATA.')
    parser.add_argument(
        'test_folder',
        type=Path,
        default=os.environ.get('MICALL_RAW_DATA',
                               Path.home() / "data/RAW_DATA"),
        help='Local RAWDATA folder that will be used to run tests.')
    parser.add_argument(
        '-s',
        '--samples_csv',
        default='test_samples.csv',
        help='CSV file with sample and run names.')
    parser.add_argument(
        '-m',
        '--min_run_name',
        help='Select all later runs, (e.g., "161201"). Overrides samples_csv.')
    parser.add_argument(
        '--pipeline_version',
        default='0-dev',
        help='version suffix for folder names')
    return parser.parse_args()

class Sample(object):
    def __init__(self, run_name, extract_num, config):
        self.run_name = run_name
        self.extract_num = extract_num
        self.config = config
        self.fastq_paths = None

    def find(self, source_folder, qai_run_names=None):
        """ Find matching samples in the source folder.

        Puts all the sample names in self.fastq_paths.
        :param str source_folder: the folder to search for samples that match
            self.extract_num
        :param set qai_run_names: a set to add the new run name to as it would
            be formatted on QAI.
        """
        run_path = os.path.join(source_folder, 'MiSeq', 'runs', self.run_name)
        if os.path.exists(run_path):
            self.run_name = run_path
        else:
            run_parts = str(self.run_name).split('.')
            if len(run_parts) < 2:
                run_parts.append('M01841')
            format_string = '%d-%b-%y' if len(run_parts[0]) == 9 else '%d-%b-%Y'
            run_date = datetime.strptime(run_parts[0], format_string)
            base_run_name = run_date.strftime('%y%m%d') + '_' + run_parts[1]
            pattern = os.path.join(source_folder,
                                   'MiSeq',
                                   'runs',
                                   base_run_name+'*',
                                   NEEDS_PROCESSING)
            matches = glob(pattern)
            if len(matches) != 1:
                raise RuntimeError('Expected one match for {}, but found: {}'.format(
                    pattern,
                    matches))
            self.run_name = os.path.dirname(matches[0])

        pattern = os.path.join(self.run_name,
                               'Data',
                               'Intensities',
                               'BaseCalls',
                               self.extract_num + '*_R1_*')
        matches = glob(pattern)
        if len(matches) == 0:
            raise RuntimeError('No matches found for ' + pattern)
        matches.sort()
        self.fastq_paths = matches
        if qai_run_names is not None:
            sample_sheet_path = os.path.join(self.run_name, 'SampleSheet.csv')
            with open(sample_sheet_path) as f:
                try:
                    sample_sheet = sample_sheet_parser(f)
                except ValueError:
                    print(f'Bad sample sheet for {self.run_name}.')
                else:
                    qai_run_name = sample_sheet['Project Name']
                    qai_run_names.add(qai_run_name)

    def setup_samples(self):
        base_run_name = os.path.basename(self.run_name)
        target_data_path = os.path.join(self.config.test_folder,
                                        'MiSeq',
                                        'runs',
                                        base_run_name,
                                        'Data',
                                        'Intensities',
                                        'BaseCalls')
        suspended_path = os.path.join(target_data_path, 'suspended')
        sample_paths = []
        for fastq1_path in self.fastq_paths:
            fastq2_path = fastq1_path.replace('_R1_', '_R2_')
            for fastq_path in (fastq1_path, fastq2_path):
                base_name = os.path.basename(fastq_path)
                target_path = os.path.join(target_data_path, base_name)
                sample_paths.append(target_path)
                if not os.path.exists(target_path):
                    suspended_fastq_path = os.path.join(suspended_path,
                                                        base_name)
                    if os.path.exists(suspended_fastq_path):
                        os.rename(suspended_fastq_path,
                                  target_path)
                    else:
                        os.symlink(fastq_path, target_path)
        return sample_paths

    def setup_run(self):
        base_run_name = os.path.basename(self.run_name)
        target_run_path = os.path.join(self.config.test_folder,
                                       'MiSeq',
                                       'runs',
                                       base_run_name)
        suspended_run_path = os.path.join(self.config.test_folder,
                                          'MiSeq',
                                          'runs',
                                          'suspended',
                                          base_run_name)
        if os.path.exists(target_run_path):
            pass
        elif os.path.exists(suspended_run_path):
            os.rename(suspended_run_path, target_run_path)
        else:
            base_calls_path = os.path.join(target_run_path,
                                           'Data',
                                           'Intensities',
                                           'BaseCalls')
            os.makedirs(base_calls_path)
            os.symlink(os.path.join(self.run_name, 'InterOp'),
                       os.path.join(target_run_path, 'InterOp'))
            for filename in ('RunInfo.xml',
                             'SampleSheet.csv',
                             'needsprocessing'):
                copy(os.path.join(self.run_name, filename),
                     os.path.join(target_run_path, filename))
        results_path = os.path.join(target_run_path,
                                    'Results',
                                    'version_' + self.config.pipeline_version)
        try:
            rmtree(results_path)
        except OSError as ex:
            if ex.errno != errno.ENOENT:
                raise
            os.makedirs(os.path.dirname(results_path), exist_ok=True)
        try:
            os.remove(os.path.join(target_run_path,
                                   ERROR_PROCESSING))
        except OSError as ex:
            if ex.errno != errno.ENOENT:
                raise
        return target_run_path

def suspend_inactive_runs(active_runs, rawdata_mount):
    active_run_names = set(map(os.path.basename, active_runs))
    local_runs = glob(os.path.join(rawdata_mount, 'MiSeq', 'runs', '*'))
    for run_path in local_runs:
        base_run_name = os.path.basename(run_path)
        if base_run_name not in active_run_names and base_run_name != 'suspended':
            suspended_path = os.path.join(rawdata_mount,
                                          'MiSeq',
                                          'runs',
                                          'suspended',
                                          base_run_name)
            os.rename(run_path, suspended_path)

def suspend_inactive_samples(run_path, active_sample_paths):
    base_calls_path = os.path.join(run_path, 'Data', 'Intensities', 'BaseCalls')
    local_samples = glob(os.path.join(base_calls_path, '*'))
    excluded = list(active_sample_paths)
    suspended_path = os.path.join(base_calls_path, 'suspended')
    excluded.append(suspended_path)
    for sample_path in local_samples:
        if sample_path not in excluded:
            if not os.path.exists(suspended_path):
                os.makedirs(suspended_path)
            os.rename(sample_path,
                      os.path.join(suspended_path, os.path.basename(sample_path)))
    try:
        os.rmdir(suspended_path)
    except OSError as ex:
        if ex.errno not in (errno.ENOTEMPTY, errno.ENOENT):
            raise

def find_run_folders(source_folder, min_run_name):
    pattern = os.path.join(source_folder, 'MiSeq', 'runs', '*')
    matches = glob(pattern)
    folder_names = [os.path.basename(match) for match in matches]
    recent_folders = [folder
                      for folder in folder_names
                      if folder >= min_run_name and folder[0].isnumeric()]
    folders_with_data = [folder
                         for folder in recent_folders
                         if os.path.exists(os.path.join(source_folder,
                                                        'MiSeq',
                                                        'runs',
                                                        folder,
                                                        NEEDS_PROCESSING))]
    return folders_with_data

def main():
    args = parse_args()
    if args.min_run_name is not None:
        print('Setting up',
              args.pipeline_version,
              'for runs after',
              args.min_run_name)
        run_folders = find_run_folders(args.old_folder, args.min_run_name)
        samples = [Sample(folder, '*', args) for folder in run_folders]
    else:
        print('Setting up',
              args.pipeline_version,
              'for runs in',
              args.samples_csv)
        with open(args.samples_csv, 'r') as samples_csv:
            samples = [Sample(row['run'], row['enum'], args)
                       for row in DictReader(samples_csv)]
    qai_run_names = set()
    for sample in samples:
        sample.find(args.old_folder, qai_run_names)
    for qai_run_name in sorted(qai_run_names):
        print('LabMiseqRun.import("{}")'.format(qai_run_name))
    samples.sort(key=lambda s: (s.run_name, s.extract_num))
    runs = []
    for run, run_samples in groupby(samples, key=attrgetter('run_name')):
        runs.append(run)
        run_name = os.path.basename(run)
        print('--' + run_name)
        sample_paths = []
        run_path = None
        for sample in run_samples:
            run_path = sample.setup_run()
            sample_paths.extend(sample.setup_samples())
            print('  ' + ', '.join(map(os.path.basename, sample.fastq_paths)))
        suspend_inactive_samples(run_path, active_sample_paths=sample_paths)
    suspend_inactive_runs(active_runs=runs, rawdata_mount=args.test_folder)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>try:
    import json
except ImportError:
    import simplejson as json

import httplib2
import httplib
import logging

from remoteobjects.dataobject import DataObject, DataObjectMetaclass
from remoteobjects import fields

userAgent = httplib2.Http()

log = logging.getLogger('remoteobjects.http')

def omit_nulls(data):
    """Strips `None` values from a dictionary or `RemoteObject` instance."""
    if not isinstance(data, dict):
        if not hasattr(data, '__dict__'):
            return str(data)
        data = dict(data.__dict__)
    for key in data.keys():
        # TODO: don't have etag in obj data in the first place?
        if data[key] is None or key == 'etag':
            del data[key]
    return data

class HttpObject(DataObject):

    """A `DataObject` that can be fetched and put over HTTP through a RESTful
    JSON API."""

    location_headers = {
        httplib.CREATED:           'Location',
        httplib.MOVED_PERMANENTLY: 'Location',
        httplib.FOUND:             'Location',
        httplib.OK:                'Content-Location',
        httplib.NOT_MODIFIED:      'Content-Location',
        httplib.NO_CONTENT:        None,
    }

    class NotFound(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the requested
        resource was not found."""
        pass

    class Unauthorized(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the requested
        resource is not available through an unauthenticated request.

        This exception corresponds to the HTTP status code 401. Thus when this
        exception is received, the caller may need to try again using the
        available authentication credentials.

        """
        pass

    class Forbidden(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the client, as
        authenticated, is not authorized to request the requested resource.

        This exception corresponds to the HTTP status code 403. Thus when this
        exception is received, nothing the caller (as currently authenticated) can
        do will make the requested resource available.

        """
        pass

    class PreconditionFailed(httplib.HTTPException):
        """An HTTPException thrown when the server reports that some of the
        conditions in a conditional request were not true.

        This exception corresponds to the HTTP status code 412. The most
        common cause of this status is an attempt to ``PUT`` a resource that
        has already changed on the server.

        """
        pass

    class RequestError(httplib.HTTPException):
        """An HTTPException thrown when the server reports an error in the
        client's request.

        This exception corresponds to the HTTP status code 400.

        """
        pass

    class ServerError(httplib.HTTPException):
        """An HTTPException thrown when the server reports an unexpected error.

        This exception corresponds to the HTTP status code 500.

        """
        pass

    class BadResponse(httplib.HTTPException):
        """An HTTPException thrown when the client receives some other
        non-success HTTP response."""
        pass

    def __init__(self, **kwargs):
        self._location = None
        super(HttpObject, self).__init__(**kwargs)

    def get_request(self, headers=None, **kwargs):
        """Returns the parameters for requesting this `RemoteObject` instance
        as a dictionary of keyword arguments suitable for passing to
        `httplib2.Http.request()`.

        Optional parameter `headers` are also included in the request as HTTP
        headers. Other optional keyword parameters are also included as
        specified.

        """
        if headers is None:
            headers = {}
        if 'accept' not in headers:
            headers['accept'] = 'application/json'

        # Use 'uri' because httplib2.request does.
        request = dict(uri=self._location, headers=headers)
        request.update(kwargs)
        return request

    @classmethod
    def get_response(cls, url, http=None, headers=None, **kwargs):
        """Requests the given URL using the specified settings.

        Optional parameter `http` is the user agent instance to use. User
        agent instances should be compatible with `httplib2.Http` instances.
        If not specified, the request is made through the instance at
        `remoteobjects.remote.userAgent`.

        Optional parameter `headers` are also included in the request as HTTP
        headers. Other optional keyword parameters are also included in the
        request as specified.

        """
        # TODO: reconcile this with get_request... which is an instance method.
        log.debug('Fetching %s', url)

        if headers is None:
            headers = {}
        if 'accept' not in headers:
            headers['accept'] = 'application/json'

        if http is None:
            http = userAgent
        response, content = http.request(uri=url, headers=headers, **kwargs)
        log.debug('Got content %r', content)

        return response, content

    @classmethod
    def raise_for_response(cls, url, response, content):
        """Raises exceptions corresponding to invalid HTTP responses that
        instances of this class can't be updated from.

        Override this method to customize the error handling behavior of
        `RemoteObject` for your target API. For example, if your API illegally
        omits ``Location`` headers from 201 Created responses, override this
        method to check for and allow them.

        """
        # Turn exceptional httplib2 responses into exceptions.
        classname = cls.__name__
        if response.status == httplib.NOT_FOUND:
            raise cls.NotFound('No such %s %s' % (classname, url))
        if response.status == httplib.UNAUTHORIZED:
            raise cls.Unauthorized('Not authorized to fetch %s %s' % (classname, url))
        if response.status == httplib.FORBIDDEN:
            raise cls.Forbidden('Forbidden from fetching %s %s' % (classname, url))
        if response.status == httplib.PRECONDITION_FAILED:
            raise cls.PreconditionFailed('Precondition failed for %s request to %s' % (classname, url))

        if response.status in (httplib.INTERNAL_SERVER_ERROR, httplib.BAD_REQUEST):
            if response.status == httplib.BAD_REQUEST:
                err_cls = cls.RequestError
            else:
                err_cls = cls.ServerError
            # Pull out an error if we can.
            content_type = response.get('content-type', '').split(';', 1)[0].strip()
            if content_type == 'text/plain':
                error = content.split('\n', 2)[0]
                raise err_cls('%d %s requesting %s %s: %s'
                    % (response.status, response.reason, classname, url,
                       error))
            raise err_cls('%d %s requesting %s %s'
                % (response.status, response.reason, classname, url))

        try:
            location_header = cls.location_headers[response.status]
        except KeyError:
            # we only expect the statuses that have location headers defined
            raise cls.BadResponse('Unexpected response requesting %s %s: %d %s'
                % (classname, url, response.status, response.reason))

        if location_header is None:
            # then there's no content-type either, so we're done
            return

        if location_header.lower() not in response:
            raise cls.BadResponse(
                "%r header missing from %d %s response requesting %s %s"
                % (location_header, response.status, response.reason,
                   classname, url))

        # check that the response body was json
        content_type = response.get('content-type', '').split(';', 1)[0].strip()
        if content_type != 'application/json':
            raise cls.BadResponse(
                'Bad response fetching %s %s: content-type is %s, not JSON'
                % (classname, url, response.get('content-type')))

    def update_from_response(self, url, response, content):
        """Adds the content of this HTTP response and message body to this
        `RemoteObject` instance.

        Use `update_from_response()` only when you would use
        `DataObject.update_from_dict()`: when decoding outside content (in
        this case an HTTP response) into an existing `RemoteObject` instance.

        If the response is not a successful response from which the
        `RemoteObject` instance can be updated, an appropriate exception will
        be raised (as determined by the instance's `raise_from_response()`
        method).

        If the response includes a new location URL in the appropriate header
        (depending on the response status), the location of the `RemoteObject`
        instance is updated as well.

        """
        self.raise_for_response(url, response, content)

        data = json.loads(content)
        self.update_from_dict(data)

        location_header = self.location_headers.get(response.status)
        if location_header is not None:
            self._location = response[location_header.lower()]

        if 'etag' in response:
            self._etag = response['etag']

    @classmethod
    def get(cls, url, http=None, **kwargs):
        """Fetches a new `RemoteObject` instance from a URL.

        Parameter `url` is the URL from which the object should be requested.
        Optional parameter `http` is the user agent object to use for
        fetching. `http` should be compatible with `httplib2.Http` instances.

        """
        response, content = cls.get_response(url, http)
        # Make a new instance and use update_from_response(), rather than
        # having a superfluous from_response() classmethod for this one call.
        self = cls()
        self.update_from_response(url, response, content)
        return self

    def post(self, obj, http=None):
        """Add another `RemoteObject` to this remote resource through an HTTP
        ``POST`` request.

        Parameter `obj` is a `RemoteObject` instance to save to this
        instance's resource. For example, this (`self`) may be a collection to
        which you want to post an asset (`obj`).

        Optional parameter `http` is the user agent object to use for posting.
        `http` should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot add %r to %r with no URL to POST to'
                % (obj, self))

        body = json.dumps(obj.to_dict(), default=omit_nulls)
        response, content = obj.get_response(self._location, http=http,
            method='POST', body=body)
        obj.update_from_response(self._location, response, content)

    def put(self, http=None):
        """Save a previously requested `RemoteObject` back to its remote
        resource through an HTTP ``PUT`` request.

        Optional `http` parameter is the user agent object to use. `http`
        objects should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot save %r with no URL to PUT to' % self)

        body = json.dumps(self.to_dict(), default=omit_nulls)

        headers = {}
        if hasattr(self, '_etag') and self._etag is not None:
            headers['if-match'] = self._etag

        response, content = self.get_response(self._location, http=http, method='PUT',
            body=body, headers=headers)
        log.debug('Yay saved my obj, now turning %r into new content', content)
        self.update_from_response(self._location, response, content)

    def delete(self, http=None):
        """Delete the remote resource represented by the `RemoteObject`
        instance through an HTTP ``DELETE`` request.

        Optional parameter `http` is the user agent object to use. `http`
        objects should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot delete %r with no URL to DELETE' % self)

        headers = {}
        if hasattr(self, '_etag') and self._etag is not None:
            headers['if-match'] = self._etag

        response, content = self.get_response(self._location, http=http,
            method='DELETE', headers=headers)
        self.raise_for_response(self._location, response, content)

        log.debug('Yay deleted the remote resource, now disconnecting %r from it', self)

        # No more resource, no more URL.
        self._location = None
        try:
            del self._etag
        except AttributeError:
            # Don't mind if there's no etag.
            pass
<commit_msg>Etags already aren't in regular instance data anymore<commit_after>try:
    import json
except ImportError:
    import simplejson as json

import httplib2
import httplib
import logging

from remoteobjects.dataobject import DataObject, DataObjectMetaclass
from remoteobjects import fields

userAgent = httplib2.Http()

log = logging.getLogger('remoteobjects.http')

def omit_nulls(data):
    """Strips `None` values from a dictionary or `RemoteObject` instance."""
    if not isinstance(data, dict):
        if not hasattr(data, '__dict__'):
            return str(data)
        data = dict(data.__dict__)
    for key in data.keys():
        if data[key] is None:
            del data[key]
    return data

class HttpObject(DataObject):

    """A `DataObject` that can be fetched and put over HTTP through a RESTful
    JSON API."""

    location_headers = {
        httplib.CREATED:           'Location',
        httplib.MOVED_PERMANENTLY: 'Location',
        httplib.FOUND:             'Location',
        httplib.OK:                'Content-Location',
        httplib.NOT_MODIFIED:      'Content-Location',
        httplib.NO_CONTENT:        None,
    }

    class NotFound(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the requested
        resource was not found."""
        pass

    class Unauthorized(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the requested
        resource is not available through an unauthenticated request.

        This exception corresponds to the HTTP status code 401. Thus when this
        exception is received, the caller may need to try again using the
        available authentication credentials.

        """
        pass

    class Forbidden(httplib.HTTPException):
        """An HTTPException thrown when the server reports that the client, as
        authenticated, is not authorized to request the requested resource.

        This exception corresponds to the HTTP status code 403. Thus when this
        exception is received, nothing the caller (as currently authenticated) can
        do will make the requested resource available.

        """
        pass

    class PreconditionFailed(httplib.HTTPException):
        """An HTTPException thrown when the server reports that some of the
        conditions in a conditional request were not true.

        This exception corresponds to the HTTP status code 412. The most
        common cause of this status is an attempt to ``PUT`` a resource that
        has already changed on the server.

        """
        pass

    class RequestError(httplib.HTTPException):
        """An HTTPException thrown when the server reports an error in the
        client's request.

        This exception corresponds to the HTTP status code 400.

        """
        pass

    class ServerError(httplib.HTTPException):
        """An HTTPException thrown when the server reports an unexpected error.

        This exception corresponds to the HTTP status code 500.

        """
        pass

    class BadResponse(httplib.HTTPException):
        """An HTTPException thrown when the client receives some other
        non-success HTTP response."""
        pass

    def __init__(self, **kwargs):
        self._location = None
        super(HttpObject, self).__init__(**kwargs)

    def get_request(self, headers=None, **kwargs):
        """Returns the parameters for requesting this `RemoteObject` instance
        as a dictionary of keyword arguments suitable for passing to
        `httplib2.Http.request()`.

        Optional parameter `headers` are also included in the request as HTTP
        headers. Other optional keyword parameters are also included as
        specified.

        """
        if headers is None:
            headers = {}
        if 'accept' not in headers:
            headers['accept'] = 'application/json'

        # Use 'uri' because httplib2.request does.
        request = dict(uri=self._location, headers=headers)
        request.update(kwargs)
        return request

    @classmethod
    def get_response(cls, url, http=None, headers=None, **kwargs):
        """Requests the given URL using the specified settings.

        Optional parameter `http` is the user agent instance to use. User
        agent instances should be compatible with `httplib2.Http` instances.
        If not specified, the request is made through the instance at
        `remoteobjects.remote.userAgent`.

        Optional parameter `headers` are also included in the request as HTTP
        headers. Other optional keyword parameters are also included in the
        request as specified.

        """
        # TODO: reconcile this with get_request... which is an instance method.
        log.debug('Fetching %s', url)

        if headers is None:
            headers = {}
        if 'accept' not in headers:
            headers['accept'] = 'application/json'

        if http is None:
            http = userAgent
        response, content = http.request(uri=url, headers=headers, **kwargs)
        log.debug('Got content %r', content)

        return response, content

    @classmethod
    def raise_for_response(cls, url, response, content):
        """Raises exceptions corresponding to invalid HTTP responses that
        instances of this class can't be updated from.

        Override this method to customize the error handling behavior of
        `RemoteObject` for your target API. For example, if your API illegally
        omits ``Location`` headers from 201 Created responses, override this
        method to check for and allow them.

        """
        # Turn exceptional httplib2 responses into exceptions.
        classname = cls.__name__
        if response.status == httplib.NOT_FOUND:
            raise cls.NotFound('No such %s %s' % (classname, url))
        if response.status == httplib.UNAUTHORIZED:
            raise cls.Unauthorized('Not authorized to fetch %s %s' % (classname, url))
        if response.status == httplib.FORBIDDEN:
            raise cls.Forbidden('Forbidden from fetching %s %s' % (classname, url))
        if response.status == httplib.PRECONDITION_FAILED:
            raise cls.PreconditionFailed('Precondition failed for %s request to %s' % (classname, url))

        if response.status in (httplib.INTERNAL_SERVER_ERROR, httplib.BAD_REQUEST):
            if response.status == httplib.BAD_REQUEST:
                err_cls = cls.RequestError
            else:
                err_cls = cls.ServerError
            # Pull out an error if we can.
            content_type = response.get('content-type', '').split(';', 1)[0].strip()
            if content_type == 'text/plain':
                error = content.split('\n', 2)[0]
                raise err_cls('%d %s requesting %s %s: %s'
                    % (response.status, response.reason, classname, url,
                       error))
            raise err_cls('%d %s requesting %s %s'
                % (response.status, response.reason, classname, url))

        try:
            location_header = cls.location_headers[response.status]
        except KeyError:
            # we only expect the statuses that have location headers defined
            raise cls.BadResponse('Unexpected response requesting %s %s: %d %s'
                % (classname, url, response.status, response.reason))

        if location_header is None:
            # then there's no content-type either, so we're done
            return

        if location_header.lower() not in response:
            raise cls.BadResponse(
                "%r header missing from %d %s response requesting %s %s"
                % (location_header, response.status, response.reason,
                   classname, url))

        # check that the response body was json
        content_type = response.get('content-type', '').split(';', 1)[0].strip()
        if content_type != 'application/json':
            raise cls.BadResponse(
                'Bad response fetching %s %s: content-type is %s, not JSON'
                % (classname, url, response.get('content-type')))

    def update_from_response(self, url, response, content):
        """Adds the content of this HTTP response and message body to this
        `RemoteObject` instance.

        Use `update_from_response()` only when you would use
        `DataObject.update_from_dict()`: when decoding outside content (in
        this case an HTTP response) into an existing `RemoteObject` instance.

        If the response is not a successful response from which the
        `RemoteObject` instance can be updated, an appropriate exception will
        be raised (as determined by the instance's `raise_from_response()`
        method).

        If the response includes a new location URL in the appropriate header
        (depending on the response status), the location of the `RemoteObject`
        instance is updated as well.

        """
        self.raise_for_response(url, response, content)

        data = json.loads(content)
        self.update_from_dict(data)

        location_header = self.location_headers.get(response.status)
        if location_header is not None:
            self._location = response[location_header.lower()]

        if 'etag' in response:
            self._etag = response['etag']

    @classmethod
    def get(cls, url, http=None, **kwargs):
        """Fetches a new `RemoteObject` instance from a URL.

        Parameter `url` is the URL from which the object should be requested.
        Optional parameter `http` is the user agent object to use for
        fetching. `http` should be compatible with `httplib2.Http` instances.

        """
        response, content = cls.get_response(url, http)
        # Make a new instance and use update_from_response(), rather than
        # having a superfluous from_response() classmethod for this one call.
        self = cls()
        self.update_from_response(url, response, content)
        return self

    def post(self, obj, http=None):
        """Add another `RemoteObject` to this remote resource through an HTTP
        ``POST`` request.

        Parameter `obj` is a `RemoteObject` instance to save to this
        instance's resource. For example, this (`self`) may be a collection to
        which you want to post an asset (`obj`).

        Optional parameter `http` is the user agent object to use for posting.
        `http` should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot add %r to %r with no URL to POST to'
                % (obj, self))

        body = json.dumps(obj.to_dict(), default=omit_nulls)
        response, content = obj.get_response(self._location, http=http,
            method='POST', body=body)
        obj.update_from_response(self._location, response, content)

    def put(self, http=None):
        """Save a previously requested `RemoteObject` back to its remote
        resource through an HTTP ``PUT`` request.

        Optional `http` parameter is the user agent object to use. `http`
        objects should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot save %r with no URL to PUT to' % self)

        body = json.dumps(self.to_dict(), default=omit_nulls)

        headers = {}
        if hasattr(self, '_etag') and self._etag is not None:
            headers['if-match'] = self._etag

        response, content = self.get_response(self._location, http=http, method='PUT',
            body=body, headers=headers)
        log.debug('Yay saved my obj, now turning %r into new content', content)
        self.update_from_response(self._location, response, content)

    def delete(self, http=None):
        """Delete the remote resource represented by the `RemoteObject`
        instance through an HTTP ``DELETE`` request.

        Optional parameter `http` is the user agent object to use. `http`
        objects should be compatible with `httplib2.Http` objects.

        """
        if getattr(self, '_location', None) is None:
            raise ValueError('Cannot delete %r with no URL to DELETE' % self)

        headers = {}
        if hasattr(self, '_etag') and self._etag is not None:
            headers['if-match'] = self._etag

        response, content = self.get_response(self._location, http=http,
            method='DELETE', headers=headers)
        self.raise_for_response(self._location, response, content)

        log.debug('Yay deleted the remote resource, now disconnecting %r from it', self)

        # No more resource, no more URL.
        self._location = None
        try:
            del self._etag
        except AttributeError:
            # Don't mind if there's no etag.
            pass
<|endoftext|>


<commit_before># flake8: noqa
import argparse
import asyncio
import logging
import os
import re
from xml.dom import minidom

from repour import asutil, exception

logger = logging.getLogger(__name__)

REPOUR_JAVA_KEY = "-DRepour_Java="
SERVICE_BUILD_CATEGORY = "SERVICE"

stdout_options = asutil.process_stdout_options
stderr_options = asutil.process_stderr_options

def get_removed_repos(work_dir, parameters):
    """
    Parses the filename of the removed repos backup file from the parameters list and if there
    is one, it reads the list of repos and returns it.
    """
    result = []

    pattern = re.compile("-DrepoRemovalBackup[ =](.+)")
    for parameter in parameters:
        m = pattern.match(parameter)
        if m is not None:
            filepath = os.path.join(work_dir, m.group(1))
            logger.debug(
                "Files and folders in the work directory:\n  %s", os.listdir(work_dir)
            )

            if os.path.exists(filepath):
                tree = minidom.parse(filepath)
                for repo_elem in tree.getElementsByTagName("repository"):
                    repo = {
                        "releases": True,
                        "snapshots": True,
                        "name": "",
                        "id": "",
                        "url": "",
                    }
                    for enabled_elem in repo_elem.getElementsByTagName("enabled"):
                        if enabled_elem.parentNode.localName in [
                            "releases",
                            "snapshots",
                        ]:
                            bool_value = enabled_elem.childNodes[0].data == "true"
                            repo[enabled_elem.parentNode.localName] = bool_value
                    for tag in ["id", "name", "url"]:
                        for elem in repo_elem.getElementsByTagName(tag):
                            if elem.childNodes:
                                repo[tag] = elem.childNodes[0].data
                    result.append(repo)
                break
            else:
                logger.info(
                    "File %s does not exist. It seems no repositories were removed",
                    filepath,
                )

    logger.info("Removed repos are: " + str(result))
    return result

def is_temp_build(adjustspec):
    """For temp build to be active, we need to both provide a 'is_temp_build' key
    in our request and its value must be true

    return: :bool: whether temp build feature enabled or not
    """
    return has_key_true(adjustspec, "tempBuild")

def is_alignment_preference(adjustspec, value):
    """For temp build to be active, we need to both provide a 'is_temp_build' key
    in our request and its value must be true

    return: :bool: whether temp build feature enabled or not
    """
    return has_key_value(adjustspec, "alignmentPreference", value)

def has_key_true(adjustspec, key):
    """Checks if the key is in the adjustspec and if the key is True.

    return: :bool: if the key is set and is True
    """
    if (key in adjustspec) and adjustspec[key] is True:
        return True
    else:
        return False

def has_key_value(adjustspec, key, value):
    """Checks if the key is in the adjustspec and if the key has given value.

    return: :bool: if the key is set and is True
    """
    if (key in adjustspec) and adjustspec[key] == value:
        return True
    else:
        return False

def get_build_version_suffix_prefix(build_category_config, temp_build_enabled):
    """Generate the prefix of version suffix (i.e. suffix prefix) from the adjust request.

    If the suffix prefix is set *AND* the one of the special build types is active then
    this function returns the value of the suffix prefix.

    Otherwise it will return None
    """
    temporary_prefix = "temporary"

    suffix_prefix = build_category_config["suffix_prefix"]

    if temp_build_enabled:
        if suffix_prefix:
            suffix_prefix = suffix_prefix + "-" + temporary_prefix
        else:
            suffix_prefix = temporary_prefix

    return suffix_prefix

def strip_temporary_from_prefix(suffix_prefix: str):
    """Strip "-temporary" portion from suffix_prefix generated by get_build_version_suffix_prefix

    Returns stripped suffix_prefix
    """
    temporary_prefix = "temporary"

    return suffix_prefix.removesuffix(temporary_prefix).rstrip("-")

def get_extra_parameters(extra_adjust_parameters, flags=("-f", "--file")):
    """
    Get the extra build configuration parameters from PNC
    If the parameters contain '<flag>=<folder>/pom.xml', then extract that folder
    and remove that <flag> option from the list of extra params.
    In PME 2.11 and PME 2.12, there is a bug where that option causes the file target/pom-manip-ext-result.json
    to be badly generated. Fixed in PME 2.13+
    See: PRODTASKS-361

    'flags' (:tuple:) specify which options to check for the extra parameters. By default it is '-f' and '--file', but can be adjusted

    Returns: tuple<list<string>, string>: list of params(minus the <flag> option), and folder where to run PME
    If '<flag>' option not used, the folder will be an empty string
    """
    subfolder = ""

    paramsString = extra_adjust_parameters.get("ALIGNMENT_PARAMETERS", None)
    if paramsString is None:
        return [], subfolder
    else:

        parser = argparse.ArgumentParser()
        parser.add_argument(*flags)
        (options, remaining_args) = parser.parse_known_args(paramsString.split())

        # remove any leading dash
        option_title = re.sub(r"^-*", "", flags[-1])

        if getattr(options, option_title) is not None:
            subfolder = getattr(options, option_title)

        return remaining_args, subfolder

def verify_folder_exists(folder, error_msg):
    """
    Checks if the folder exists and it is a folder. If not, a exception.CommandError is thrown
    """
    if os.path.exists(folder) and os.path.isdir(folder):
        # all ok! do nothing!
        pass
    else:
        if not os.path.exists(folder):
            raise exception.CommandError(
                error_msg, [], 10, stdout=error_msg, stderr=error_msg
            )
        else:
            raise exception.CommandError(
                error_msg, [], 10, stdout=error_msg, stderr=error_msg
            )

def get_jvm_from_extra_parameters(extra_parameters):
    """
    If repour JVM option specified, return the option value. Otherwise return None
    """

    for parameter in extra_parameters:

        if REPOUR_JAVA_KEY in parameter:
            return parameter.replace(REPOUR_JAVA_KEY, "")
    else:
        return None

def get_param_value(name, params1, params2=[], params3=[]):
    """
    Searches for the given param name in given arrays and returns the value of the first occurrence. The param name must
    match completely, so you have to pass it along with "-D" if it is in the arrays.
    """
    for params in [params1, params2, params3]:
        for param in params:
            if param.startswith(name + "="):
                return param.replace(name + "=", "")
    return None

async def print_java_version(java_bin_dir=""):

    if java_bin_dir and java_bin_dir.endswith("/"):
        command = java_bin_dir + "java"
    elif java_bin_dir:
        command = java_bin_dir + "/java"
    else:
        command = "java"

    expect_ok = asutil.expect_ok_closure()
    output = await expect_ok(
        cmd=[command, "-version"],
        desc="Failed getting Java version",
        cwd=".",
        stdout=stdout_options["text"],
        stderr=stderr_options["stdout"],
        print_cmd=True,
    )
    logger.info(output)

async def generate_user_context():
    """For now, returns a string of key:value,key:value"""
    current_task = asyncio.current_task()
    return "log-user-id:{},log-request-context:{},log-process-context:{},log-expires:{},log-tmp:{},trace-id:{},span-id:{}".format(
        current_task.mdc["userId"],
        current_task.mdc["requestContext"],
        current_task.mdc["processContext"],
        current_task.mdc["expires"],
        current_task.mdc["tmp"],
        current_task.mdc["traceId"],
        current_task.mdc["spanId"],
    )
<commit_msg>[NCL-7340] Fix typos in mdc values<commit_after># flake8: noqa
import argparse
import asyncio
import logging
import os
import re
from xml.dom import minidom

from repour import asutil, exception

logger = logging.getLogger(__name__)

REPOUR_JAVA_KEY = "-DRepour_Java="
SERVICE_BUILD_CATEGORY = "SERVICE"

stdout_options = asutil.process_stdout_options
stderr_options = asutil.process_stderr_options

def get_removed_repos(work_dir, parameters):
    """
    Parses the filename of the removed repos backup file from the parameters list and if there
    is one, it reads the list of repos and returns it.
    """
    result = []

    pattern = re.compile("-DrepoRemovalBackup[ =](.+)")
    for parameter in parameters:
        m = pattern.match(parameter)
        if m is not None:
            filepath = os.path.join(work_dir, m.group(1))
            logger.debug(
                "Files and folders in the work directory:\n  %s", os.listdir(work_dir)
            )

            if os.path.exists(filepath):
                tree = minidom.parse(filepath)
                for repo_elem in tree.getElementsByTagName("repository"):
                    repo = {
                        "releases": True,
                        "snapshots": True,
                        "name": "",
                        "id": "",
                        "url": "",
                    }
                    for enabled_elem in repo_elem.getElementsByTagName("enabled"):
                        if enabled_elem.parentNode.localName in [
                            "releases",
                            "snapshots",
                        ]:
                            bool_value = enabled_elem.childNodes[0].data == "true"
                            repo[enabled_elem.parentNode.localName] = bool_value
                    for tag in ["id", "name", "url"]:
                        for elem in repo_elem.getElementsByTagName(tag):
                            if elem.childNodes:
                                repo[tag] = elem.childNodes[0].data
                    result.append(repo)
                break
            else:
                logger.info(
                    "File %s does not exist. It seems no repositories were removed",
                    filepath,
                )

    logger.info("Removed repos are: " + str(result))
    return result

def is_temp_build(adjustspec):
    """For temp build to be active, we need to both provide a 'is_temp_build' key
    in our request and its value must be true

    return: :bool: whether temp build feature enabled or not
    """
    return has_key_true(adjustspec, "tempBuild")

def is_alignment_preference(adjustspec, value):
    """For temp build to be active, we need to both provide a 'is_temp_build' key
    in our request and its value must be true

    return: :bool: whether temp build feature enabled or not
    """
    return has_key_value(adjustspec, "alignmentPreference", value)

def has_key_true(adjustspec, key):
    """Checks if the key is in the adjustspec and if the key is True.

    return: :bool: if the key is set and is True
    """
    if (key in adjustspec) and adjustspec[key] is True:
        return True
    else:
        return False

def has_key_value(adjustspec, key, value):
    """Checks if the key is in the adjustspec and if the key has given value.

    return: :bool: if the key is set and is True
    """
    if (key in adjustspec) and adjustspec[key] == value:
        return True
    else:
        return False

def get_build_version_suffix_prefix(build_category_config, temp_build_enabled):
    """Generate the prefix of version suffix (i.e. suffix prefix) from the adjust request.

    If the suffix prefix is set *AND* the one of the special build types is active then
    this function returns the value of the suffix prefix.

    Otherwise it will return None
    """
    temporary_prefix = "temporary"

    suffix_prefix = build_category_config["suffix_prefix"]

    if temp_build_enabled:
        if suffix_prefix:
            suffix_prefix = suffix_prefix + "-" + temporary_prefix
        else:
            suffix_prefix = temporary_prefix

    return suffix_prefix

def strip_temporary_from_prefix(suffix_prefix: str):
    """Strip "-temporary" portion from suffix_prefix generated by get_build_version_suffix_prefix

    Returns stripped suffix_prefix
    """
    temporary_prefix = "temporary"

    return suffix_prefix.removesuffix(temporary_prefix).rstrip("-")

def get_extra_parameters(extra_adjust_parameters, flags=("-f", "--file")):
    """
    Get the extra build configuration parameters from PNC
    If the parameters contain '<flag>=<folder>/pom.xml', then extract that folder
    and remove that <flag> option from the list of extra params.
    In PME 2.11 and PME 2.12, there is a bug where that option causes the file target/pom-manip-ext-result.json
    to be badly generated. Fixed in PME 2.13+
    See: PRODTASKS-361

    'flags' (:tuple:) specify which options to check for the extra parameters. By default it is '-f' and '--file', but can be adjusted

    Returns: tuple<list<string>, string>: list of params(minus the <flag> option), and folder where to run PME
    If '<flag>' option not used, the folder will be an empty string
    """
    subfolder = ""

    paramsString = extra_adjust_parameters.get("ALIGNMENT_PARAMETERS", None)
    if paramsString is None:
        return [], subfolder
    else:

        parser = argparse.ArgumentParser()
        parser.add_argument(*flags)
        (options, remaining_args) = parser.parse_known_args(paramsString.split())

        # remove any leading dash
        option_title = re.sub(r"^-*", "", flags[-1])

        if getattr(options, option_title) is not None:
            subfolder = getattr(options, option_title)

        return remaining_args, subfolder

def verify_folder_exists(folder, error_msg):
    """
    Checks if the folder exists and it is a folder. If not, a exception.CommandError is thrown
    """
    if os.path.exists(folder) and os.path.isdir(folder):
        # all ok! do nothing!
        pass
    else:
        if not os.path.exists(folder):
            raise exception.CommandError(
                error_msg, [], 10, stdout=error_msg, stderr=error_msg
            )
        else:
            raise exception.CommandError(
                error_msg, [], 10, stdout=error_msg, stderr=error_msg
            )

def get_jvm_from_extra_parameters(extra_parameters):
    """
    If repour JVM option specified, return the option value. Otherwise return None
    """

    for parameter in extra_parameters:

        if REPOUR_JAVA_KEY in parameter:
            return parameter.replace(REPOUR_JAVA_KEY, "")
    else:
        return None

def get_param_value(name, params1, params2=[], params3=[]):
    """
    Searches for the given param name in given arrays and returns the value of the first occurrence. The param name must
    match completely, so you have to pass it along with "-D" if it is in the arrays.
    """
    for params in [params1, params2, params3]:
        for param in params:
            if param.startswith(name + "="):
                return param.replace(name + "=", "")
    return None

async def print_java_version(java_bin_dir=""):

    if java_bin_dir and java_bin_dir.endswith("/"):
        command = java_bin_dir + "java"
    elif java_bin_dir:
        command = java_bin_dir + "/java"
    else:
        command = "java"

    expect_ok = asutil.expect_ok_closure()
    output = await expect_ok(
        cmd=[command, "-version"],
        desc="Failed getting Java version",
        cwd=".",
        stdout=stdout_options["text"],
        stderr=stderr_options["stdout"],
        print_cmd=True,
    )
    logger.info(output)

async def generate_user_context():
    """For now, returns a string of key:value,key:value"""
    current_task = asyncio.current_task()
    return "log-user-id:{},log-request-context:{},log-process-context:{},log-expires:{},log-tmp:{},trace-id:{},span-id:{}".format(
        current_task.mdc["userId"],
        current_task.mdc["requestContext"],
        current_task.mdc["processContext"],
        current_task.mdc["expires"],
        current_task.mdc["tmp"],
        current_task.mdc["trace_id"],
        current_task.mdc["span_id"],
    )
<|endoftext|>


<commit_before># This file is used to configure the behavior of pytest when using the Astropy
# test infrastructure.

from astropy.version import version as astropy_version
if astropy_version < '3.0':
    # With older versions of Astropy, we actually need to import the pytest
    # plugins themselves in order to make them discoverable by pytest.
    from astropy.tests.pytest_plugins import *
else:
    # As of Astropy 3.0, the pytest plugins provided by Astropy are
    # automatically made available when Astropy is installed. This means it's
    # not necessary to import them here, but we still need to import global
    # variables that are used for configuration.
    from astropy.tests.plugins.display import PYTEST_HEADER_MODULES, TESTED_VERSIONS

from astropy.tests.helper import enable_deprecations_as_exceptions

enable_deprecations_as_exceptions()

# Uncomment and customize the following lines to add/remove entries from
# the list of packages for which version numbers are displayed when running
# the tests. Making it pass for KeyError is essential in some cases when
# the package uses other astropy affiliated packages.
try:
    PYTEST_HEADER_MODULES['Astropy'] = 'astropy'
    PYTEST_HEADER_MODULES['astropy-healpix'] = 'astropy_healpix'
    PYTEST_HEADER_MODULES['Cython'] = 'cython'
    del PYTEST_HEADER_MODULES['h5py']
    del PYTEST_HEADER_MODULES['Matplotlib']
except (NameError, KeyError):  # NameError is needed to support Astropy < 1.0
    pass

# Uncomment the following lines to display the version number of the
# package rather than the version number of Astropy in the top line when
# running the tests.
import os

# This is to figure out the affiliated package version, rather than
# using Astropy's
try:
    from .version import version
except ImportError:
    version = 'dev'

try:
    packagename = os.path.basename(os.path.dirname(__file__))
    TESTED_VERSIONS[packagename] = version
except NameError:   # Needed to support Astropy <= 1.0.0
    pass
<commit_msg>Minor conftest cleanup<commit_after># This file is used to configure the behavior of pytest when using the Astropy
# test infrastructure.
import os
from distutils.version import LooseVersion
from astropy.version import version as astropy_version

if LooseVersion(astropy_version) < LooseVersion('2.0.3'):
    # Astropy is not compatible with the standalone plugins prior this while
    # astroquery requires them, so we need this workaround. This will mess
    # up the test header, but everything else will work.
    from astropy.tests.pytest_plugins import (PYTEST_HEADER_MODULES,
                                              TESTED_VERSIONS)
elif astropy_version < '3.0':
    # With older versions of Astropy, we actually need to import the pytest
    # plugins themselves in order to make them discoverable by pytest.
    from astropy.tests.pytest_plugins import *
else:
    # As of Astropy 3.0, the pytest plugins provided by Astropy are
    # automatically made available when Astropy is installed. This means it's
    # not necessary to import them here, but we still need to import global
    # variables that are used for configuration.
    from astropy.tests.plugins.display import PYTEST_HEADER_MODULES, TESTED_VERSIONS

from astropy.tests.helper import enable_deprecations_as_exceptions

enable_deprecations_as_exceptions()

# Uncomment and customize the following lines to add/remove entries from
# the list of packages for which version numbers are displayed when running
# the tests. Making it pass for KeyError is essential in some cases when
# the package uses other astropy affiliated packages.
try:
    PYTEST_HEADER_MODULES['Astropy'] = 'astropy'
    PYTEST_HEADER_MODULES['astropy-healpix'] = 'astropy_healpix'
    PYTEST_HEADER_MODULES['Cython'] = 'cython'
    del PYTEST_HEADER_MODULES['h5py']
    del PYTEST_HEADER_MODULES['Matplotlib']
except (NameError, KeyError):  # NameError is needed to support Astropy < 1.0
    pass

# Uncomment the following lines to display the version number of the
# package rather than the version number of Astropy in the top line when
# running the tests.

# This is to figure out the reproject's version, rather than
# using Astropy's
from .version import version, astropy_helpers_version

packagename = os.path.basename(os.path.dirname(__file__))
TESTED_VERSIONS[packagename] = version
TESTED_VERSIONS['astropy_helpers'] = astropy_helpers_version
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
import inflect
import util
import parse
import sys
import random
from sets import Set
from textblob import TextBlob, Word
from nltk.corpus import wordnet as wn
import string
from textblob.taggers import NLTKTagger

def ask_questions(filename, numQuestions, debug=False):
    nltk_tagger = NLTKTagger()
    p = inflect.engine()
    try:
        doc = util.load_article(filename)
        database = parse.basic_parse(doc)
    except:
        print "Invalid filename: " + filename
        sys.exit()
    art = ("the", "an", "a", "The")
    pairs = []
    result = []
    for key in database:
        if key[0] in string.punctuation: continue
        for entry in database[key]:
            pairs += [(key, entry)]
    if (numQuestions > len(pairs)):
        selected = random.sample(pairs, len(pairs))
    else:
        selected = random.sample(pairs, numQuestions)
    for (key, value) in selected:
        entry = database[key][value]
        """syn = util.synonyms(value, False)
        if (len(syn) != 0):
            print "replaced " + value
            value = random.sample(syn, 1)[0]"""
        key_plural = util.is_plural(key)
        is_verb = "Are" if key_plural else "Is"
        verb_phrase = entry['relation']
        #print verb_phrase
        try:
            verb = verb_phrase[0]
        except: verb = "related"
        question = ""
        if (verb == "is"):
            if not value.startswith(art):
                value = p.a(value)
            question = "%s %s %s?" % (is_verb, key, value)
        elif (verb == "has"):
            has_verb = "Do" if key_plural else "Does"
            if not value.startswith(art):
                value = p.a(value)
            question = "%s %s have %s?" % (has_verb, key, value)
        else:
            blob = TextBlob(" ".join(verb_phrase), pos_tagger=nltk_tagger)
            tags = blob.pos_tags
            verbs = []
            for w,tag in blob.pos_tags:
                if tag[0] == "V":
                    verbs += [(w, tag)]
            action = [w for (w, tag) in verbs if tag != "VBZ"]
            det_option = " the" if not key[0].isupper() else ""

            for (w, tag) in verbs:
                question3 = "What does%s %s %s?" % (det_option, key,  w.lemmatize("v"))
                #print question3
            det_option = " the" if not key[0].isupper() else ""
            question2 = "%s%s %s %s %s?" % (is_verb, det_option, key, blob, value)
            #print question2
            question = "%s%s %s related to %s?" % (is_verb, det_option, key, value)
        if (question != ""):
            print question
            result += [question]
    return result

if __name__ == "__main__":

    try:
        filename = sys.argv[1]
        numQuestions = int(sys.argv[2])
    except:
        #print "Invalid filename or numQuestions, using default values"
        filename = 'set4/a1'
        numQuestions = 30

    #print "Generating %d questions from %s" % (numQuestions, filename)
    ask_questions(filename, numQuestions)
<commit_msg>Fix to work on POS-tagged relations<commit_after>#!/usr/bin/env python2.7
import inflect
import util
import parse
import sys
import random
from sets import Set
from textblob import TextBlob, Word
from nltk.corpus import wordnet as wn
import string
from textblob.taggers import NLTKTagger

def ask_questions(filename, numQuestions, debug=False):
    nltk_tagger = NLTKTagger()
    p = inflect.engine()
    try:
        doc = util.load_article(filename)
        database = parse.basic_parse(doc)
    except:
        print "Invalid filename: " + filename
        sys.exit()
    art = ("the", "an", "a", "The")
    pairs = []
    result = []
    for key in database:
        if key[0] in string.punctuation: continue
        for entry in database[key]:
            pairs += [(key, entry)]
    if (numQuestions > len(pairs)):
        selected = random.sample(pairs, len(pairs))
    else:
        selected = random.sample(pairs, numQuestions)
    for (key, value) in selected:
        entry = database[key][value]
        """syn = util.synonyms(value, False)
        if (len(syn) != 0):
            print "replaced " + value
            value = random.sample(syn, 1)[0]"""
        key_plural = util.is_plural(key)
        is_verb = "Are" if key_plural else "Is"
        verb_phrase = [verb[0] for verb in entry['relation']]
        #print verb_phrase
        try:
            verb = verb_phrase[0]
        except: verb = "related"
        question = ""
        if (verb == "is"):
            if not value.startswith(art):
                value = p.a(value)
            question = "%s %s %s?" % (is_verb, key, value)
        elif (verb == "has"):
            has_verb = "Do" if key_plural else "Does"
            if not value.startswith(art):
                value = p.a(value)
            question = "%s %s have %s?" % (has_verb, key, value)
        else:
            blob = TextBlob(" ".join(verb_phrase), pos_tagger=nltk_tagger)
            tags = blob.pos_tags
            verbs = []
            for w,tag in blob.pos_tags:
                if tag[0] == "V":
                    verbs += [(w, tag)]
            action = [w for (w, tag) in verbs if tag != "VBZ"]
            det_option = " the" if not key[0].isupper() else ""
            #print " ".join(verb_phrase)
            #print value
            for (w, tag) in verbs:
                if w.lemmatize("v") != "be":
                    if tag == "VBD": does_verb = "did"
                    else: does_verb = "does"
                    question3 = "What %s%s %s %s?" % (det_option.capitalize(), does_verb, key, w.lemmatize("v"))
                else:
                    question3 = "%s%s %s %s?" % (w, det_option, key, value)
                #print question3
            det_option = " the" if not key[0].isupper() else ""
            question2 = "%s%s %s %s %s?" % (is_verb, det_option, key, blob, value)
            #print question2
            question = "%s%s %s related to %s?" % (is_verb, det_option, key, value)
        if (question != ""):
            print question
            result += [question]
    return result

if __name__ == "__main__":

    try:
        filename = sys.argv[1]
        numQuestions = int(sys.argv[2])
    except:
        #print "Invalid filename or numQuestions, using default values"
        filename = 'data/set1/a1.txt'
        numQuestions = 30

    #print "Generating %d questions from %s" % (numQuestions, filename)
    ask_questions(filename, numQuestions)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibClassify daemon.

FIXME: the code below requires collection table to be updated to add column:
   clsMETHOD_fk mediumint(9) unsigned NOT NULL,
This is not clean and should be fixed.
"""

import sys
import time
import os

from invenio.dbquery import run_sql
from invenio.bibtask import task_init, write_message, get_datetime, \
    task_set_option, task_get_option, task_update_status, \
    task_update_progress
from invenio.bibclassify_engine import get_keywords_from_text
from invenio.config import CFG_BINDIR, CFG_ETCDIR, CFG_TMPDIR, CFG_PATH_PDFTOTEXT
from invenio.intbitset import intbitset
from invenio.search_engine import get_collection_reclist
from invenio.bibdocfile import BibRecDocs

def get_recids_foreach_ontology():
    """Returns an array containing hash objects containing the collection, its
    corresponding ontology and the records belonging to the given collection."""

    rec_onts = []
    res = run_sql("""SELECT clsMETHOD.name, last_updated, collection.name
        FROM clsMETHOD JOIN collection_clsMETHOD ON clsMETHOD.id=id_clsMETHOD
        JOIN collection ON id_collection=collection.id""")
    for ontology, date_last_run, collection in res:
        recs = get_collection_reclist(collection)
        if recs:
            if not date_last_run:
                date_last_run = '0000-00-00'
            modified_records = intbitset(run_sql("SELECT id FROM bibrec WHERE modification_date >=%s", (date_last_run, )))
            recs &= modified_records
            if recs:
                rec_onts.append({
                    'ontology' : ontology,
                    'collection' : collection,
                    'recIDs' : recs
                })
    return rec_onts

def update_date_of_last_run():
    """
    Update bibclassify daemon table information about last run time.
    """
    run_sql("UPDATE clsMETHOD SET last_updated=NOW()")

def task_run_core():
    """Runs anayse_documents for each ontology,collection,record ids set."""
    outfilename = CFG_TMPDIR + "/bibclassifyd_%s.xml" % time.strftime("%Y%m%dH%M%S", time.localtime())
    outfiledesc = open(outfilename, "w")
    coll_counter = 0
    print >> outfiledesc, """<?xml version="1.0" encoding="UTF-8"?>"""
    print >> outfiledesc, """<collection xmlns="http://www.loc.gov/MARC21/slim">"""
    for onto_rec in get_recids_foreach_ontology():
        write_message('Applying taxonomy %s to collection %s (%s records)' % (onto_rec['ontology'], onto_rec['collection'], len(onto_rec['recIDs'])))
        if onto_rec['recIDs']:
            coll_counter += analyse_documents(onto_rec['recIDs'], onto_rec['ontology'], onto_rec['collection'], outfilename, outfiledesc)
    print >> outfiledesc, '</collection>'
    outfiledesc.close()
    if coll_counter:
        cmd = "%s/bibupload -n -c '%s' " % (CFG_BINDIR, outfilename)
        errcode = 0
        try:
            errcode = os.system(cmd)
        except OSError, exc:
            print 'command' + cmd + ' failed  ', exc
        if errcode != 0:
            write_message("WARNING, %s failed, error code is %s" % (cmd, errcode))
            return 0
    update_date_of_last_run()
    return 1

def analyse_documents(recs, ontology, collection, outfilename, outfiledesc):
    """For each collection, parse the documents attached to the records in collection with the corresponding ontology."""

    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    did_something = False
    counter = 1
    amax = len(recs)
    # store running time:
    # see which records we'll have to process:
    #recIDs = get_recIDs_of_modified_records_since_last_run()
    temp_text = None
    if recs:
        # process records:
        cmd = None
        path = None
        temp_text = CFG_TMPDIR + '/bibclassify.pdftotext.' + str(os.getpid())
        for rec in recs:
            bibdocfiles = BibRecDocs(rec).list_latest_files()
            found_one_pdf = False
            for bibdocfile in bibdocfiles:
                if bibdocfile.get_format() == '.pdf':
                    found_one_pdf = True
            if found_one_pdf:
                did_something = True
                print >> outfiledesc, '<record>'
                print >> outfiledesc, """<controlfield tag="001">%(recID)s</controlfield>""" % ({'recID':rec})
                for f in bibdocfiles:
                    if f.get_format() == '.pdf':

                        cmd = "%s '%s' '%s'" % (CFG_PATH_PDFTOTEXT, f.get_full_path(), temp_text)
                    else:
                        write_message("Can't parse file %s." % f.get_full_path(), verbose=3)
                        continue
                    errcode = os.system(cmd)
                    if errcode != 0 or not os.path.exists("%s" % temp_text):
                        write_message("Error while executing command %s Error code was: %s " % (cmd, errcode))
                    write_message('Generating keywords for %s' % f.get_full_path())

def get_keywords_from_text(text_lines, output_mode="text",
    output_limit=CFG_BIBCLASSIFY_DEFAULT_OUTPUT_NUMBER, spires=False,
    match_mode="full", no_cache=False, with_author_keywords=False):

                    print >> outfiledesc, get_keywords_from_text(temp_text, CFG_ETCDIR + '/bibclassify/' + ontology + '.rdf', 2, 70, 25, 0, False, verbose=0, ontology=ontology)
                print >> outfiledesc, '</record>'
            task_update_progress("Done %s of %s for collection  %s." % (counter, amax, collection))
            counter += 1
    else:
        write_message("Nothing to be done, move along")
    return did_something

def cleanup_tmp():
    """Remove old temporary files created by this module"""
    for f in os.listdir(CFG_TMPDIR):
        if 'bibclassify' in f: os.remove(CFG_TMPDIR + '/' +f)

def main():
    """Constructs the bibclassifyd bibtask."""
    cleanup_tmp()
    task_init(authorization_action='runbibclassify',
              authorization_msg="BibClassify Task Submission",
              description="""Examples:
        %s -u admin
""" % (sys.argv[0],),
              version="",
              task_run_fnc = task_run_core)

if __name__ == '__main__':
    main()

# FIXME: one can have more than one ontologies in clsMETHOD.
#    bibclassifyd -w HEP,Pizza

# FIXME: add more CLI options like bibindex ones, e.g.
#    bibclassifyd -a -i 10-20

# FIXME: outfiledesc can be multiple files, e.g. when processing
#    100000 records it is good to store results by 1000 records
#    (see oaiharvest)
<commit_msg>Fixed syntactic error.<commit_after># -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
BibClassify daemon.

FIXME: the code below requires collection table to be updated to add column:
   clsMETHOD_fk mediumint(9) unsigned NOT NULL,
This is not clean and should be fixed.
"""

import sys
import time
import os

from invenio.dbquery import run_sql
from invenio.bibtask import task_init, write_message, get_datetime, \
    task_set_option, task_get_option, task_update_status, \
    task_update_progress
from invenio.bibclassify_engine import get_keywords_from_text
from invenio.config import CFG_BINDIR, CFG_ETCDIR, CFG_TMPDIR, CFG_PATH_PDFTOTEXT
from invenio.intbitset import intbitset
from invenio.search_engine import get_collection_reclist
from invenio.bibdocfile import BibRecDocs

def get_recids_foreach_ontology():
    """Returns an array containing hash objects containing the collection, its
    corresponding ontology and the records belonging to the given collection."""

    rec_onts = []
    res = run_sql("""SELECT clsMETHOD.name, last_updated, collection.name
        FROM clsMETHOD JOIN collection_clsMETHOD ON clsMETHOD.id=id_clsMETHOD
        JOIN collection ON id_collection=collection.id""")
    for ontology, date_last_run, collection in res:
        recs = get_collection_reclist(collection)
        if recs:
            if not date_last_run:
                date_last_run = '0000-00-00'
            modified_records = intbitset(run_sql("SELECT id FROM bibrec WHERE modification_date >=%s", (date_last_run, )))
            recs &= modified_records
            if recs:
                rec_onts.append({
                    'ontology' : ontology,
                    'collection' : collection,
                    'recIDs' : recs
                })
    return rec_onts

def update_date_of_last_run():
    """
    Update bibclassify daemon table information about last run time.
    """
    run_sql("UPDATE clsMETHOD SET last_updated=NOW()")

def task_run_core():
    """Runs anayse_documents for each ontology,collection,record ids set."""
    outfilename = CFG_TMPDIR + "/bibclassifyd_%s.xml" % time.strftime("%Y%m%dH%M%S", time.localtime())
    outfiledesc = open(outfilename, "w")
    coll_counter = 0
    print >> outfiledesc, """<?xml version="1.0" encoding="UTF-8"?>"""
    print >> outfiledesc, """<collection xmlns="http://www.loc.gov/MARC21/slim">"""
    for onto_rec in get_recids_foreach_ontology():
        write_message('Applying taxonomy %s to collection %s (%s records)' % (onto_rec['ontology'], onto_rec['collection'], len(onto_rec['recIDs'])))
        if onto_rec['recIDs']:
            coll_counter += analyse_documents(onto_rec['recIDs'], onto_rec['ontology'], onto_rec['collection'], outfilename, outfiledesc)
    print >> outfiledesc, '</collection>'
    outfiledesc.close()
    if coll_counter:
        cmd = "%s/bibupload -n -c '%s' " % (CFG_BINDIR, outfilename)
        errcode = 0
        try:
            errcode = os.system(cmd)
        except OSError, exc:
            print 'command' + cmd + ' failed  ', exc
        if errcode != 0:
            write_message("WARNING, %s failed, error code is %s" % (cmd, errcode))
            return 0
    update_date_of_last_run()
    return 1

def analyse_documents(recs, ontology, collection, outfilename, outfiledesc):
    """For each collection, parse the documents attached to the records in collection with the corresponding ontology."""

    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    did_something = False
    counter = 1
    amax = len(recs)
    # store running time:
    # see which records we'll have to process:
    #recIDs = get_recIDs_of_modified_records_since_last_run()
    temp_text = None
    if recs:
        # process records:
        cmd = None
        path = None
        temp_text = CFG_TMPDIR + '/bibclassify.pdftotext.' + str(os.getpid())
        for rec in recs:
            bibdocfiles = BibRecDocs(rec).list_latest_files()
            found_one_pdf = False
            for bibdocfile in bibdocfiles:
                if bibdocfile.get_format() == '.pdf':
                    found_one_pdf = True
            if found_one_pdf:
                did_something = True
                print >> outfiledesc, '<record>'
                print >> outfiledesc, """<controlfield tag="001">%(recID)s</controlfield>""" % ({'recID':rec})
                for f in bibdocfiles:
                    if f.get_format() == '.pdf':

                        cmd = "%s '%s' '%s'" % (CFG_PATH_PDFTOTEXT, f.get_full_path(), temp_text)
                    else:
                        write_message("Can't parse file %s." % f.get_full_path(), verbose=3)
                        continue
                    errcode = os.system(cmd)
                    if errcode != 0 or not os.path.exists("%s" % temp_text):
                        write_message("Error while executing command %s Error code was: %s " % (cmd, errcode))
                    write_message('Generating keywords for %s' % f.get_full_path())
                    print >> outfiledesc, get_keywords_from_text(temp_text, CFG_ETCDIR + '/bibclassify/' + ontology + '.rdf', 2, 70, 25, 0, False, verbose=0, ontology=ontology)
                print >> outfiledesc, '</record>'
            task_update_progress("Done %s of %s for collection  %s." % (counter, amax, collection))
            counter += 1
    else:
        write_message("Nothing to be done, move along")
    return did_something

def cleanup_tmp():
    """Remove old temporary files created by this module"""
    for f in os.listdir(CFG_TMPDIR):
        if 'bibclassify' in f: os.remove(CFG_TMPDIR + '/' +f)

def main():
    """Constructs the bibclassifyd bibtask."""
    cleanup_tmp()
    task_init(authorization_action='runbibclassify',
              authorization_msg="BibClassify Task Submission",
              description="""Examples:
        %s -u admin
""" % (sys.argv[0],),
              version="",
              task_run_fnc = task_run_core)

if __name__ == '__main__':
    main()

# FIXME: one can have more than one ontologies in clsMETHOD.
#    bibclassifyd -w HEP,Pizza

# FIXME: add more CLI options like bibindex ones, e.g.
#    bibclassifyd -a -i 10-20

# FIXME: outfiledesc can be multiple files, e.g. when processing
#    100000 records it is good to store results by 1000 records
#    (see oaiharvest)
<|endoftext|>


<commit_before># Created By: Virgil Dupras
# Created On: 2006/02/21
# Copyright 2010 Hardcoded Software (http://www.hardcoded.net)

# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license

import os
import os.path as op

from . import io
from . import conflict

def open_if_filename(infile, mode='rb'):
    """
    infile can be either a string or a file-like object.
    if it is a string, a file will be opened with mode.
    Returns a tuple (shouldbeclosed,infile) infile is a file object
    """
    if isinstance(infile, basestring):
        return (open(infile, mode), True)
    else:
        return (infile, False)

class FileOrPath(object):
    def __init__(self, file_or_path, mode='rb'):
        self.file_or_path = file_or_path
        self.mode = mode
        self.mustclose = False
        self.fp = None

    def __enter__(self):
        self.fp, self.mustclose = open_if_filename(self.file_or_path, self.mode)
        return self.fp

    def __exit__(self, exc_type, exc_value, traceback):
        if self.fp and self.mustclose:
            self.fp.close()

def clean_empty_dirs(path, deleteself=False, files_to_delete=[]):
    """Recursively delete empty dirs in directory. 'directory' is deleted only
    if empty and 'deleteself' is True.
    Returns the list of delete paths.
    files_to_delete: The name is clear enough. However, files in
        this list will ONLY be deleted if it makes the directory deletable
        thereafter (In other words, if the directory contains files not in the
        list, NO file will be deleted)
    """
    result = []
    subdirs = [name for name in io.listdir(path) if io.isdir(path + name)]
    for subdir in subdirs:
        result.extend(clean_empty_dirs(path + subdir, True, files_to_delete))
    if deleteself and delete_if_empty(path, files_to_delete):
        result.append(unicode(path))
    return result

@io.log_io_error
def delete_if_empty(path, files_to_delete=[]):
    ''' Deletes the directory at 'path' if it is empty or if it only contains files_to_delete.
    '''
    if not io.exists(path) or not io.isdir(path):
        return
    contents = io.listdir(path)
    if any(name for name in contents if (name not in files_to_delete) or io.isdir(path + name)):
        return False
    for name in contents:
        io.remove(path + name)
    io.rmdir(path)
    return True

def _move_or_copy(operation, source_path, dest_path):
    ''' Use move() or copy() to move and copy file with the conflict management, but without the
        slowness of the fs system.
    '''
    if io.isdir(dest_path):
        dest_path = dest_path + source_path[-1]
    if io.exists(dest_path):
        filename = dest_path[-1]
        dest_dir_path = dest_path[:-1]
        newname = conflict.get_conflicted_name(io.listdir(dest_dir_path), filename)
        dest_path = dest_dir_path + newname
    operation(source_path, dest_path)

def move(source_path, dest_path):
    _move_or_copy(io.move, source_path, dest_path)

def copy(source_path, dest_path):
    _move_or_copy(io.copy, source_path, dest_path)

def modified_after(first_path, second_path):
    """Returns True if first_path's mtime is higher than second_path's mtime."""
    try:
        first_mtime = io.stat(first_path).st_mtime
    except EnvironmentError:
        return False
    try:
        second_mtime = io.stat(second_path).st_mtime
    except EnvironmentError:
        return True
    return first_mtime > second_mtime<commit_msg>Added hsutil.files.find_in_path()<commit_after># Created By: Virgil Dupras
# Created On: 2006/02/21
# Copyright 2010 Hardcoded Software (http://www.hardcoded.net)

# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license

import os
import os.path as op

from . import io
from . import conflict

def open_if_filename(infile, mode='rb'):
    """
    infile can be either a string or a file-like object.
    if it is a string, a file will be opened with mode.
    Returns a tuple (shouldbeclosed,infile) infile is a file object
    """
    if isinstance(infile, basestring):
        return (open(infile, mode), True)
    else:
        return (infile, False)

class FileOrPath(object):
    def __init__(self, file_or_path, mode='rb'):
        self.file_or_path = file_or_path
        self.mode = mode
        self.mustclose = False
        self.fp = None

    def __enter__(self):
        self.fp, self.mustclose = open_if_filename(self.file_or_path, self.mode)
        return self.fp

    def __exit__(self, exc_type, exc_value, traceback):
        if self.fp and self.mustclose:
            self.fp.close()

def clean_empty_dirs(path, deleteself=False, files_to_delete=[]):
    """Recursively delete empty dirs in directory. 'directory' is deleted only
    if empty and 'deleteself' is True.
    Returns the list of delete paths.
    files_to_delete: The name is clear enough. However, files in
        this list will ONLY be deleted if it makes the directory deletable
        thereafter (In other words, if the directory contains files not in the
        list, NO file will be deleted)
    """
    result = []
    subdirs = [name for name in io.listdir(path) if io.isdir(path + name)]
    for subdir in subdirs:
        result.extend(clean_empty_dirs(path + subdir, True, files_to_delete))
    if deleteself and delete_if_empty(path, files_to_delete):
        result.append(unicode(path))
    return result

@io.log_io_error
def delete_if_empty(path, files_to_delete=[]):
    ''' Deletes the directory at 'path' if it is empty or if it only contains files_to_delete.
    '''
    if not io.exists(path) or not io.isdir(path):
        return
    contents = io.listdir(path)
    if any(name for name in contents if (name not in files_to_delete) or io.isdir(path + name)):
        return False
    for name in contents:
        io.remove(path + name)
    io.rmdir(path)
    return True

def _move_or_copy(operation, source_path, dest_path):
    ''' Use move() or copy() to move and copy file with the conflict management, but without the
        slowness of the fs system.
    '''
    if io.isdir(dest_path):
        dest_path = dest_path + source_path[-1]
    if io.exists(dest_path):
        filename = dest_path[-1]
        dest_dir_path = dest_path[:-1]
        newname = conflict.get_conflicted_name(io.listdir(dest_dir_path), filename)
        dest_path = dest_dir_path + newname
    operation(source_path, dest_path)

def move(source_path, dest_path):
    _move_or_copy(io.move, source_path, dest_path)

def copy(source_path, dest_path):
    _move_or_copy(io.copy, source_path, dest_path)

def modified_after(first_path, second_path):
    """Returns True if first_path's mtime is higher than second_path's mtime."""
    try:
        first_mtime = io.stat(first_path).st_mtime
    except EnvironmentError:
        return False
    try:
        second_mtime = io.stat(second_path).st_mtime
    except EnvironmentError:
        return True
    return first_mtime > second_mtime

def find_in_path(name, path=None):
    """Search for `name` in all directories of `path` and return the absolute path of the first
    occurrence. If `path` is None, $PATH is used.
    """
    if path is None:
        path = os.environ['PATH']
    paths = path.split(os.pathsep)
    for path in paths:
        if op.exists(op.join(path, name)):
            return op.join(path, name)
    return None<|endoftext|>