<commit_before>import os
import yaml
import json
import re
import httplib2
import urllib
import logging
import socket

import teuthology
from teuthology.config import config

log = logging.getLogger(__name__)

def main(args):
    if args['--verbose']:
        teuthology.log.setLevel(logging.DEBUG)

    archive_base = os.path.abspath(os.path.expanduser(args['--archive']))
    save = not args['--no-save']
    reporter = ResultsReporter(archive_base, base_uri=args['--server'],
                               save=save, refresh=args['--refresh'])
    run = args['--run']
    job = args['--job']
    dead = args['--dead']
    if dead and len(run) == 1 and job:
        for job_id in job:
            try_push_job_info(dict(name=run[0], job_id=job_id, status='dead'))
    elif len(run) == 1 and job:
        reporter.report_jobs(run[0], job)
    elif run and len(run) > 1:
        reporter.report_runs(run)
    elif run:
        reporter.report_run(run[0])
    elif args['--all-runs']:
        reporter.report_all_runs()

class RequestFailedError(RuntimeError):
    def __init__(self, uri, resp, content):
        self.uri = uri
        self.status = resp.status
        self.reason = resp.reason
        self.content = content
        try:
            self.content_obj = json.loads(content)
            self.message = self.content_obj['message']
        except ValueError:
            self.message = self.content

    def __str__(self):
        templ = "Request to {uri} failed with status {status}: {reason}: {message}"  # noqa

        return templ.format(
            uri=self.uri,
            status=self.status,
            reason=self.reason,
            message=self.message,
        )

class ResultsSerializer(object):
    """
    This class exists to poke around in the archive directory doing things like
    assembling lists of test runs, lists of their jobs, and merging sets of job
    YAML files together to form JSON objects.
    """
    yamls = ('orig.config.yaml', 'config.yaml', 'info.yaml', 'summary.yaml')

    def __init__(self, archive_base):
        self.archive_base = archive_base

    def json_for_job(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together to
        create a JSON object.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A JSON object.
        """
        job_archive_dir = os.path.join(self.archive_base,
                                       run_name,
                                       job_id)
        job_info = {}
        for yaml_name in self.yamls:
            yaml_path = os.path.join(job_archive_dir, yaml_name)
            if not os.path.exists(yaml_path):
                continue
            with file(yaml_path) as yaml_file:
                partial_info = yaml.safe_load(yaml_file)
                if partial_info is not None:
                    job_info.update(partial_info)

        if 'job_id' not in job_info:
            job_info['job_id'] = job_id

        if pretty:
            job_json = json.dumps(job_info, sort_keys=True, indent=4)
        else:
            job_json = json.dumps(job_info)

        return job_json

    def jobs_for_run(self, run_name):
        """
        Given a run name, look on the filesystem for directories containing job
        information, and return a dict mapping job IDs to job directories.

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        archive_dir = os.path.join(self.archive_base, run_name)
        if not os.path.isdir(archive_dir):
            return {}
        jobs = {}
        for item in os.listdir(archive_dir):
            if not re.match('\d+$', item):
                continue
            job_id = item
            job_dir = os.path.join(archive_dir, job_id)
            if os.path.isdir(job_dir):
                jobs[job_id] = job_dir
        return jobs

    def running_jobs_for_run(self, run_name):
        """
        Like jobs_for_run(), but only returns jobs with no summary.yaml

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        jobs = self.jobs_for_run(run_name)
        for job_id in jobs.keys():
            if os.path.exists(os.path.join(jobs[job_id], 'summary.yaml')):
                jobs.pop(job_id)
        return jobs

    @property
    def all_runs(self):
        """
        Look in the base archive directory for all test runs. Return a list of
        their names.
        """
        archive_base = self.archive_base
        if not os.path.isdir(archive_base):
            return []
        runs = []
        for run_name in os.listdir(archive_base):
            if not os.path.isdir(os.path.join(archive_base, run_name)):
                continue
            runs.append(run_name)
        return runs

class ResultsReporter(object):
    last_run_file = 'last_successful_run'

    def __init__(self, archive_base, base_uri=None, save=False, refresh=False,
                 timeout=20):
        self.archive_base = archive_base
        self.base_uri = base_uri or config.results_server
        if self.base_uri:
            self.base_uri = self.base_uri.rstrip('/')
        self.serializer = ResultsSerializer(archive_base)
        self.save_last_run = save
        self.refresh = refresh
        self.timeout = timeout

    def _do_request(self, uri, method, json_):
        """
        Perform an actual HTTP request on a given URI. If the request was not
        successful and the reason was *not* that the object already exists,
        raise a RequestFailedError.
        """
        # Use urllib.quote() to escape things like spaces. Pass safe=':/' to
        # avoid it mangling http:// etc.
        uri = urllib.quote(uri, safe=':/')
        response, content = self.http.request(
            uri, method, json_, headers={'content-type': 'application/json'},
        )
        log.debug("{method} to {uri}: {status}".format(
            method=method,
            uri=uri,
            status=response.status,
        ))

        try:
            content_obj = json.loads(content)
        except ValueError:
            content_obj = {}

        message = content_obj.get('message', '')

        if response.status != 200 and not message.endswith('already exists'):
            raise RequestFailedError(uri, response, content)

        return response.status, message, content

    def post_json(self, uri, json_):
        """
        call self._do_request(uri, 'POST', json_)
        """
        return self._do_request(uri, 'POST', json_)

    def put_json(self, uri, json_):
        """
        call self._do_request(uri, 'PUT', json_)
        """
        return self._do_request(uri, 'PUT', json_)

    def report_all_runs(self):
        """
        Report *all* runs in self.archive_dir to the results server.
        """
        all_runs = self.serializer.all_runs
        last_run = self.last_run
        if self.save_last_run and last_run and last_run in all_runs:
            next_index = all_runs.index(last_run) + 1
            runs = all_runs[next_index:]
        else:
            runs = all_runs
        return self.report_runs(runs)

    def report_runs(self, run_names):
        """
        Report several runs to the results server.

        :param run_names: The names of the runs.
        """
        num_runs = len(run_names)
        num_jobs = 0
        log.info("Posting %s runs", num_runs)
        for run in run_names:
            job_count = self.report_run(run)
            num_jobs += job_count
            if self.save_last_run:
                self.last_run = run
        del self.last_run
        log.info("Total: %s jobs in %s runs", num_jobs, len(run_names))

    def create_run(self, run_name):
        """
        Create a run on the results server.

        :param run_name: The name of the run.
        :returns:        The result of self.post_json()
        """
        run_uri = "{base}/runs/".format(base=self.base_uri, name=run_name)
        run_json = json.dumps({'name': run_name})
        return self.post_json(run_uri, run_json)

    def report_run(self, run_name):
        """
        Report a single run to the results server.

        :param run_name: The name of the run.
        :returns:        The number of jobs reported.
        """
        jobs = self.serializer.jobs_for_run(run_name)
        log.info("{name} {jobs} jobs".format(
            name=run_name,
            jobs=len(jobs),
        ))
        if jobs:
            if not self.refresh:
                status, msg, content = self.create_run(run_name)
                if status != 200:
                    log.info("    already present; skipped")
                    return 0
            self.report_jobs(run_name, jobs.keys())
        elif not jobs:
            log.debug("    no jobs; skipped")
        return len(jobs)

    def report_jobs(self, run_name, job_ids):
        """
        Report several jobs to the results server.

        :param run_name: The name of the run.
        :param job_ids:  The jobs' ids
        """
        for job_id in job_ids:
            self.report_job(run_name, job_id)

    def report_job(self, run_name, job_id, job_json=None):
        """
        Report a single job to the results server.

        :param run_name: The name of the run. The run must already exist.
        :param job_id:   The job's id
        :param job_json: The job's JSON object. Optional - if not present, we
                         look at the archive.
        """
        run_uri = "{base}/runs/{name}/jobs/".format(
            base=self.base_uri, name=run_name,)
        if job_json is None:
            job_json = self.serializer.json_for_job(run_name, job_id)
        status, msg, content = self.post_json(run_uri, job_json)

        if msg.endswith('already exists'):
            job_uri = os.path.join(run_uri, job_id, '')
            status, msg, content = self.put_json(job_uri, job_json)
        return job_id

    @property
    def last_run(self):
        """
        The last run to be successfully reported.
        """
        if hasattr(self, '__last_run'):
            return self.__last_run
        elif os.path.exists(self.last_run_file):
            with file(self.last_run_file) as f:
                self.__last_run = f.read().strip()
            return self.__last_run

    @last_run.setter
    def last_run(self, run_name):
        self.__last_run = run_name
        with file(self.last_run_file, 'w') as f:
            f.write(run_name)

    @last_run.deleter
    def last_run(self):
        self.__last_run = None
        if os.path.exists(self.last_run_file):
            os.remove(self.last_run_file)

    @property
    def http(self):
        if hasattr(self, '__http'):
            return self.__http
        self.__http = httplib2.Http(timeout=self.timeout)
        return self.__http

def create_run(run_name, base_uri=None):
    """
    Create a run on the results server. If it already exists, just smile and be
    happy.

    :param run_name: The name of the run.
    :param base_uri: The endpoint of the results server. If you leave it out
                     ResultsReporter will ask teuthology.config.
    :returns:         True if the run was successfully created.
    """
    # We are using archive_base='' here because we KNOW the serializer isn't
    # needed for this codepath.
    reporter = ResultsReporter(archive_base='', base_uri=base_uri)
    status, msg, content = reporter.create_run(run_name)
    return (status == 200 or msg.endswith('already exists'))

def push_job_info(run_name, job_id, job_info, base_uri=None):
    """
    Push a job's info (example: ctx.config) to the results server.

    :param run_name: The name of the run.
    :param job_id:   The job's id
    :param job_info: A dict containing the job's information.
    :param base_uri: The endpoint of the results server. If you leave it out
                     ResultsReporter will ask teuthology.config.
    """
    # We are using archive_base='' here because we KNOW the serializer isn't
    # needed for this codepath.
    job_json = json.dumps(job_info)
    reporter = ResultsReporter(archive_base='')
    reporter.report_job(run_name, job_id, job_json)

def try_push_job_info(job_config, extra_info=None):
    """
    Wrap push_job_info, gracefully doing nothing if:
        A RequestFailedError is raised
        A socket.error is raised
        config.results_server is not set
        config['job_id'] is not present or is None

    :param job_config: The ctx.config object to push
    :param extra_info: Optional second dict to push
    """
    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to push results"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return
    elif job_config.get('job_id') is None:
        log.warning('No job_id found; not reporting results')
        return

    run_name = job_config['name']
    job_id = job_config['job_id']

    if extra_info is not None:
        job_info = extra_info.copy()
        job_info.update(job_config)
    else:
        job_info = job_config

    try:
        log.info("Pushing job info to %s", config.results_server)
        push_job_info(run_name, job_id, job_info)
    except (RequestFailedError, socket.error, httplib2.ServerNotFoundError):
        log.exception("Could not report results to %s" %
                      config.results_server)
<commit_msg>Port from httplib2 to requests module<commit_after>import os
import yaml
import json
import re
import requests
import logging
import socket

import teuthology
from teuthology.config import config

# Don't need to see connection pool INFO messages
logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
    logging.WARNING)

log = logging.getLogger(__name__)

def main(args):
    if args['--verbose']:
        teuthology.log.setLevel(logging.DEBUG)

    archive_base = os.path.abspath(os.path.expanduser(args['--archive']))
    save = not args['--no-save']
    reporter = ResultsReporter(archive_base, base_uri=args['--server'],
                               save=save, refresh=args['--refresh'])
    run = args['--run']
    job = args['--job']
    dead = args['--dead']
    if dead and len(run) == 1 and job:
        for job_id in job:
            try_push_job_info(dict(name=run[0], job_id=job_id, status='dead'))
    elif len(run) == 1 and job:
        reporter.report_jobs(run[0], job)
    elif run and len(run) > 1:
        reporter.report_runs(run)
    elif run:
        reporter.report_run(run[0])
    elif args['--all-runs']:
        reporter.report_all_runs()

class ResultsSerializer(object):
    """
    This class exists to poke around in the archive directory doing things like
    assembling lists of test runs, lists of their jobs, and merging sets of job
    YAML files together to form JSON objects.
    """
    yamls = ('orig.config.yaml', 'config.yaml', 'info.yaml', 'summary.yaml')

    def __init__(self, archive_base):
        self.archive_base = archive_base

    def json_for_job(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together to
        create a JSON object.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A JSON object.
        """
        job_archive_dir = os.path.join(self.archive_base,
                                       run_name,
                                       job_id)
        job_info = {}
        for yaml_name in self.yamls:
            yaml_path = os.path.join(job_archive_dir, yaml_name)
            if not os.path.exists(yaml_path):
                continue
            with file(yaml_path) as yaml_file:
                partial_info = yaml.safe_load(yaml_file)
                if partial_info is not None:
                    job_info.update(partial_info)

        if 'job_id' not in job_info:
            job_info['job_id'] = job_id

        if pretty:
            job_json = json.dumps(job_info, sort_keys=True, indent=4)
        else:
            job_json = json.dumps(job_info)

        return job_json

    def jobs_for_run(self, run_name):
        """
        Given a run name, look on the filesystem for directories containing job
        information, and return a dict mapping job IDs to job directories.

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        archive_dir = os.path.join(self.archive_base, run_name)
        if not os.path.isdir(archive_dir):
            return {}
        jobs = {}
        for item in os.listdir(archive_dir):
            if not re.match('\d+$', item):
                continue
            job_id = item
            job_dir = os.path.join(archive_dir, job_id)
            if os.path.isdir(job_dir):
                jobs[job_id] = job_dir
        return jobs

    def running_jobs_for_run(self, run_name):
        """
        Like jobs_for_run(), but only returns jobs with no summary.yaml

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        jobs = self.jobs_for_run(run_name)
        for job_id in jobs.keys():
            if os.path.exists(os.path.join(jobs[job_id], 'summary.yaml')):
                jobs.pop(job_id)
        return jobs

    @property
    def all_runs(self):
        """
        Look in the base archive directory for all test runs. Return a list of
        their names.
        """
        archive_base = self.archive_base
        if not os.path.isdir(archive_base):
            return []
        runs = []
        for run_name in os.listdir(archive_base):
            if not os.path.isdir(os.path.join(archive_base, run_name)):
                continue
            runs.append(run_name)
        return runs

class ResultsReporter(object):
    last_run_file = 'last_successful_run'

    def __init__(self, archive_base, base_uri=None, save=False, refresh=False,
                 timeout=20):
        self.archive_base = archive_base
        self.base_uri = base_uri or config.results_server
        if self.base_uri:
            self.base_uri = self.base_uri.rstrip('/')
        self.serializer = ResultsSerializer(archive_base)
        self.save_last_run = save
        self.refresh = refresh
        self.timeout = timeout

    def report_all_runs(self):
        """
        Report *all* runs in self.archive_dir to the results server.
        """
        all_runs = self.serializer.all_runs
        last_run = self.last_run
        if self.save_last_run and last_run and last_run in all_runs:
            next_index = all_runs.index(last_run) + 1
            runs = all_runs[next_index:]
        else:
            runs = all_runs
        return self.report_runs(runs)

    def report_runs(self, run_names):
        """
        Report several runs to the results server.

        :param run_names: The names of the runs.
        """
        num_runs = len(run_names)
        num_jobs = 0
        log.info("Posting %s runs", num_runs)
        for run in run_names:
            job_count = self.report_run(run)
            num_jobs += job_count
            if self.save_last_run:
                self.last_run = run
        del self.last_run
        log.info("Total: %s jobs in %s runs", num_jobs, len(run_names))

    def report_run(self, run_name):
        """
        Report a single run to the results server.

        :param run_name: The name of the run.
        :returns:        The number of jobs reported.
        """
        jobs = self.serializer.jobs_for_run(run_name)
        log.info("{name} {jobs} jobs".format(
            name=run_name,
            jobs=len(jobs),
        ))
        if jobs:
            if not self.refresh:
                response = requests.head("{base}/runs/{name}/".format(
                    base=self.base_uri, name=run_name))
                if response.status_code == 200:
                    log.info("    already present; skipped")
                    return 0
            self.report_jobs(run_name, jobs.keys())
        elif not jobs:
            log.debug("    no jobs; skipped")
        return len(jobs)

    def report_jobs(self, run_name, job_ids):
        """
        Report several jobs to the results server.

        :param run_name: The name of the run.
        :param job_ids:  The jobs' ids
        """
        for job_id in job_ids:
            self.report_job(run_name, job_id)

    def report_job(self, run_name, job_id, job_json=None):
        """
        Report a single job to the results server.

        :param run_name: The name of the run. The run must already exist.
        :param job_id:   The job's id
        :param job_json: The job's JSON object. Optional - if not present, we
                         look at the archive.
        """
        run_uri = "{base}/runs/{name}/jobs/".format(
            base=self.base_uri, name=run_name,)
        if job_json is None:
            job_json = self.serializer.json_for_job(run_name, job_id)
        response = requests.post(run_uri, job_json)

        if response.status_code == 200:
            return job_id

        msg = response.json.get('message', '')
        if msg and msg.endswith('already exists'):
            job_uri = os.path.join(run_uri, job_id, '')
            response = requests.put(job_uri, job_json)
            response.raise_for_status()
        elif msg:
            log.error(
                "POST to {uri} failed with status {status}: {msg}".format(
                    uri=run_uri,
                    status=response.status_code,
                    msg=msg,
                ))

        return job_id

    @property
    def last_run(self):
        """
        The last run to be successfully reported.
        """
        if hasattr(self, '__last_run'):
            return self.__last_run
        elif os.path.exists(self.last_run_file):
            with file(self.last_run_file) as f:
                self.__last_run = f.read().strip()
            return self.__last_run

    @last_run.setter
    def last_run(self, run_name):
        self.__last_run = run_name
        with file(self.last_run_file, 'w') as f:
            f.write(run_name)

    @last_run.deleter
    def last_run(self):
        self.__last_run = None
        if os.path.exists(self.last_run_file):
            os.remove(self.last_run_file)

def push_job_info(run_name, job_id, job_info, base_uri=None):
    """
    Push a job's info (example: ctx.config) to the results server.

    :param run_name: The name of the run.
    :param job_id:   The job's id
    :param job_info: A dict containing the job's information.
    :param base_uri: The endpoint of the results server. If you leave it out
                     ResultsReporter will ask teuthology.config.
    """
    # We are using archive_base='' here because we KNOW the serializer isn't
    # needed for this codepath.
    job_json = json.dumps(job_info)
    reporter = ResultsReporter(archive_base='')
    reporter.report_job(run_name, job_id, job_json)

def try_push_job_info(job_config, extra_info=None):
    """
    Wrap push_job_info, gracefully doing nothing if:
        Anything inheriting from requests.exceptions.RequestException is raised
        A socket.error is raised
        config.results_server is not set
        config['job_id'] is not present or is None

    :param job_config: The ctx.config object to push
    :param extra_info: Optional second dict to push
    """
    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to push results"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return
    elif job_config.get('job_id') is None:
        log.warning('No job_id found; not reporting results')
        return

    run_name = job_config['name']
    job_id = job_config['job_id']

    if extra_info is not None:
        job_info = extra_info.copy()
        job_info.update(job_config)
    else:
        job_info = job_config

    try:
        log.info("Pushing job info to %s", config.results_server)
        push_job_info(run_name, job_id, job_info)
    except (requests.exceptions.RequestException, socket.error):
        log.exception("Could not report results to %s" %
                      config.results_server)
<|endoftext|>


<commit_before>import logging
import os
import subprocess
import sys
import tempfile
import time
import yaml

from datetime import datetime

from teuthology import setup_log_file
from . import beanstalk
from . import report
from . import safepath
from .config import config as teuth_config
from .exceptions import BranchNotFoundError
from .kill import kill_job
from .misc import read_config
from .repo_utils import fetch_qa_suite, fetch_teuthology

log = logging.getLogger(__name__)
start_time = datetime.utcnow()
restart_file_path = '/tmp/teuthology-restart-workers'

def need_restart():
    if not os.path.exists(restart_file_path):
        return False
    file_mtime = datetime.utcfromtimestamp(os.path.getmtime(restart_file_path))
    if file_mtime > start_time:
        return True
    else:
        return False

def restart():
    log.info('Restarting...')
    args = sys.argv[:]
    args.insert(0, sys.executable)
    os.execv(sys.executable, args)

def install_except_hook():
    """
    Install an exception hook that first logs any uncaught exception, then
    raises it.
    """
    def log_exception(exc_type, exc_value, exc_traceback):
        if not issubclass(exc_type, KeyboardInterrupt):
            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
                                                         exc_traceback))
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
    sys.excepthook = log_exception

def main(ctx):
    loglevel = logging.INFO
    if ctx.verbose:
        loglevel = logging.DEBUG
    log.setLevel(loglevel)

    log_file_path = os.path.join(ctx.log_dir, 'worker.{tube}.{pid}'.format(
        pid=os.getpid(), tube=ctx.tube,))
    setup_log_file(log, log_file_path)

    install_except_hook()

    if not os.path.isdir(ctx.archive_dir):
        sys.exit("{prog}: archive directory must exist: {path}".format(
            prog=os.path.basename(sys.argv[0]),
            path=ctx.archive_dir,
        ))
    else:
        teuth_config.archive_base = ctx.archive_dir

    read_config(ctx)

    connection = beanstalk.connect()
    beanstalk.watch_tube(connection, ctx.tube)
    result_proc = None

    while True:
        # Check to see if we have a teuthology-results process hanging around
        # and if so, read its return code so that it can exit.
        if result_proc is not None and result_proc.poll() is not None:
            log.debug("teuthology-results exited with code: %s",
                      result_proc.returncode)
            result_proc = None

        if need_restart():
            restart()

        job = connection.reserve(timeout=60)
        if job is None:
            continue

        # bury the job so it won't be re-run if it fails
        job.bury()
        log.info('Reserved job %d', job.jid)
        log.info('Config is: %s', job.body)
        job_config = yaml.safe_load(job.body)

        job_config['job_id'] = str(job.jid)
        safe_archive = safepath.munge(job_config['name'])
        job_config['worker_log'] = log_file_path
        archive_path_full = os.path.join(
            ctx.archive_dir, safe_archive, str(job.jid))
        job_config['archive_path'] = archive_path_full

        # If the teuthology branch was not specified, default to master and
        # store that value.
        teuthology_branch = job_config.get('teuthology_branch', 'master')
        job_config['teuthology_branch'] = teuthology_branch

        try:
            teuth_path = fetch_teuthology(branch=teuthology_branch)
            # For the teuthology tasks, we look for suite_branch, and if we
            # don't get that, we look for branch, and fall back to 'master'.
            # last-in-suite jobs don't have suite_branch or branch set.
            ceph_branch = job_config.get('branch', 'master')
            suite_branch = job_config.get('suite_branch', ceph_branch)
            job_config['suite_path'] = fetch_qa_suite(suite_branch)
        except BranchNotFoundError:
            log.exception(
                "Branch not found; throwing job away")
            # Optionally, we could mark the job as dead, but we don't have a
            # great way to express why it is dead.
            report.try_delete_jobs(job_config['name'],
                                   job_config['job_id'])
            continue

        teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
        if not os.path.isdir(teuth_bin_path):
            raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
                               (teuthology_branch, teuth_bin_path))

        if job_config.get('last_in_suite'):
            if teuth_config.results_server:
                report.try_delete_jobs(job_config['name'],
                                       job_config['job_id'])
            log.info('Generating results email for %s', job_config['name'])
            args = [
                os.path.join(teuth_bin_path, 'teuthology-results'),
                '--timeout',
                str(job_config.get('results_timeout', 32400)),
                '--email',
                job_config['email'],
                '--archive-dir',
                os.path.join(ctx.archive_dir, safe_archive),
                '--name',
                job_config['name'],
            ]
            # Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to
            # make sure that it will continue to run if this worker process
            # dies (e.g. because of a restart)
            result_proc = subprocess.Popen(args=args, preexec_fn=os.setpgrp)
            log.info("teuthology-results PID: %s", result_proc.pid)
        else:
            log.info('Creating archive dir %s', archive_path_full)
            safepath.makedirs(ctx.archive_dir, safe_archive)
            log.info('Running job %d', job.jid)
            run_job(job_config, teuth_bin_path)
        job.delete()

def run_with_watchdog(process, job_config):
    job_start_time = datetime.utcnow()

    # Only push the information that's relevant to the watchdog, to save db
    # load
    job_info = dict(
        name=job_config['name'],
        job_id=job_config['job_id'],
    )

    # Sleep once outside of the loop to avoid double-posting jobs
    time.sleep(teuth_config.watchdog_interval)
    symlink_worker_log(job_config['worker_log'], job_config['archive_path'])
    while process.poll() is None:
        # Kill jobs that have been running longer than the global max
        run_time = datetime.utcnow() - job_start_time
        total_seconds = run_time.days * 60 * 60 * 24 + run_time.seconds
        if total_seconds > teuth_config.max_job_time:
            log.warning("Job ran longer than {max}s. Killing...".format(
                max=teuth_config.max_job_time))
            kill_job(job_info['name'], job_info['job_id'],
                     teuth_config.archive_base)

        report.try_push_job_info(job_info, dict(status='running'))
        time.sleep(teuth_config.watchdog_interval)

    # The job finished. Let's make sure paddles knows.
    branches_sans_reporting = ('argonaut', 'bobtail', 'cuttlefish', 'dumpling')
    if job_config.get('teuthology_branch') in branches_sans_reporting:
        # The job ran with a teuthology branch that may not have the reporting
        # feature. Let's call teuthology-report (which will be from the master
        # branch) to report the job manually.
        cmd = "teuthology-report -v -D -r {run_name} -j {job_id}".format(
            run_name=job_info['name'],
            job_id=job_info['job_id'])
        try:
            log.info("Executing %s" % cmd)
            report_proc = subprocess.Popen(cmd, shell=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.STDOUT)
            while report_proc.poll() is None:
                for line in report_proc.stdout.readlines():
                    log.info(line.strip())
                time.sleep(1)
            log.info("Reported results via the teuthology-report command")
        except Exception:
            log.exception("teuthology-report failed")
    else:
        # Let's make sure that paddles knows the job is finished. We don't know
        # the status, but if it was a pass or fail it will have already been
        # reported to paddles. In that case paddles ignores the 'dead' status.
        # If the job was killed, paddles will use the 'dead' status.
        report.try_push_job_info(job_info, dict(status='dead'))

def run_job(job_config, teuth_bin_path):
    suite_path = job_config['suite_path']
    arg = [
        os.path.join(teuth_bin_path, 'teuthology'),
    ]
    # The following is for compatibility with older schedulers, from before we
    # started merging the contents of job_config['config'] into job_config
    # itself.
    if 'config' in job_config:
        inner_config = job_config.pop('config')
        if not isinstance(inner_config, dict):
            log.warn("run_job: job_config['config'] isn't a dict, it's a %s",
                     str(type(inner_config)))
        else:
            job_config.update(inner_config)

    if job_config['verbose']:
        arg.append('-v')

    arg.extend([
        '--lock',
        '--block',
        '--owner', job_config['owner'],
        '--archive', job_config['archive_path'],
        '--name', job_config['name'],
    ])
    if job_config['description'] is not None:
        arg.extend(['--description', job_config['description']])
    arg.append('--')

    with tempfile.NamedTemporaryFile(prefix='teuthology-worker.',
                                     suffix='.tmp',) as tmp:
        yaml.safe_dump(data=job_config, stream=tmp)
        tmp.flush()
        arg.append(tmp.name)
        env = os.environ.copy()
        python_path = env.get('PYTHONPATH', '')
        python_path = ':'.join([suite_path, python_path]).strip(':')
        env['PYTHONPATH'] = python_path
        p = subprocess.Popen(args=arg, env=env)
        log.info("Job archive: %s", job_config['archive_path'])
        log.info("Job PID: %s", str(p.pid))

        if teuth_config.results_server:
            log.info("Running with watchdog")
            try:
                run_with_watchdog(p, job_config)
            except Exception:
                log.exception("run_with_watchdog had an unhandled exception")
                raise
        else:
            log.info("Running without watchdog")
            # This sleep() is to give the child time to start up and create the
            # archive dir.
            time.sleep(5)
            symlink_worker_log(job_config['worker_log'],
                               job_config['archive_path'])
            p.wait()

        if p.returncode != 0:
            log.error('Child exited with code %d', p.returncode)
        else:
            log.info('Success!')

def symlink_worker_log(worker_log_path, archive_dir):
    try:
        log.debug("Worker log: %s", worker_log_path)
        os.symlink(worker_log_path, os.path.join(archive_dir, 'worker.log'))
    except Exception:
        log.exception("Failed to symlink worker log")
<commit_msg>Attempt fetches on init, before running jobs<commit_after>import logging
import os
import subprocess
import sys
import tempfile
import time
import yaml

from datetime import datetime

from teuthology import setup_log_file
from . import beanstalk
from . import report
from . import safepath
from .config import config as teuth_config
from .exceptions import BranchNotFoundError
from .kill import kill_job
from .misc import read_config
from .repo_utils import fetch_qa_suite, fetch_teuthology

log = logging.getLogger(__name__)
start_time = datetime.utcnow()
restart_file_path = '/tmp/teuthology-restart-workers'

def need_restart():
    if not os.path.exists(restart_file_path):
        return False
    file_mtime = datetime.utcfromtimestamp(os.path.getmtime(restart_file_path))
    if file_mtime > start_time:
        return True
    else:
        return False

def restart():
    log.info('Restarting...')
    args = sys.argv[:]
    args.insert(0, sys.executable)
    os.execv(sys.executable, args)

def install_except_hook():
    """
    Install an exception hook that first logs any uncaught exception, then
    raises it.
    """
    def log_exception(exc_type, exc_value, exc_traceback):
        if not issubclass(exc_type, KeyboardInterrupt):
            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
                                                         exc_traceback))
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
    sys.excepthook = log_exception

def main(ctx):
    loglevel = logging.INFO
    if ctx.verbose:
        loglevel = logging.DEBUG
    log.setLevel(loglevel)

    log_file_path = os.path.join(ctx.log_dir, 'worker.{tube}.{pid}'.format(
        pid=os.getpid(), tube=ctx.tube,))
    setup_log_file(log, log_file_path)

    install_except_hook()

    if not os.path.isdir(ctx.archive_dir):
        sys.exit("{prog}: archive directory must exist: {path}".format(
            prog=os.path.basename(sys.argv[0]),
            path=ctx.archive_dir,
        ))
    else:
        teuth_config.archive_base = ctx.archive_dir

    read_config(ctx)

    connection = beanstalk.connect()
    beanstalk.watch_tube(connection, ctx.tube)
    result_proc = None

    fetch_teuthology('master')
    fetch_qa_suite('master')

    while True:
        # Check to see if we have a teuthology-results process hanging around
        # and if so, read its return code so that it can exit.
        if result_proc is not None and result_proc.poll() is not None:
            log.debug("teuthology-results exited with code: %s",
                      result_proc.returncode)
            result_proc = None

        if need_restart():
            restart()

        job = connection.reserve(timeout=60)
        if job is None:
            continue

        # bury the job so it won't be re-run if it fails
        job.bury()
        log.info('Reserved job %d', job.jid)
        log.info('Config is: %s', job.body)
        job_config = yaml.safe_load(job.body)

        job_config['job_id'] = str(job.jid)
        safe_archive = safepath.munge(job_config['name'])
        job_config['worker_log'] = log_file_path
        archive_path_full = os.path.join(
            ctx.archive_dir, safe_archive, str(job.jid))
        job_config['archive_path'] = archive_path_full

        # If the teuthology branch was not specified, default to master and
        # store that value.
        teuthology_branch = job_config.get('teuthology_branch', 'master')
        job_config['teuthology_branch'] = teuthology_branch

        try:
            teuth_path = fetch_teuthology(branch=teuthology_branch)
            # For the teuthology tasks, we look for suite_branch, and if we
            # don't get that, we look for branch, and fall back to 'master'.
            # last-in-suite jobs don't have suite_branch or branch set.
            ceph_branch = job_config.get('branch', 'master')
            suite_branch = job_config.get('suite_branch', ceph_branch)
            job_config['suite_path'] = fetch_qa_suite(suite_branch)
        except BranchNotFoundError:
            log.exception(
                "Branch not found; throwing job away")
            # Optionally, we could mark the job as dead, but we don't have a
            # great way to express why it is dead.
            report.try_delete_jobs(job_config['name'],
                                   job_config['job_id'])
            continue

        teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
        if not os.path.isdir(teuth_bin_path):
            raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
                               (teuthology_branch, teuth_bin_path))

        if job_config.get('last_in_suite'):
            if teuth_config.results_server:
                report.try_delete_jobs(job_config['name'],
                                       job_config['job_id'])
            log.info('Generating results email for %s', job_config['name'])
            args = [
                os.path.join(teuth_bin_path, 'teuthology-results'),
                '--timeout',
                str(job_config.get('results_timeout', 32400)),
                '--email',
                job_config['email'],
                '--archive-dir',
                os.path.join(ctx.archive_dir, safe_archive),
                '--name',
                job_config['name'],
            ]
            # Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to
            # make sure that it will continue to run if this worker process
            # dies (e.g. because of a restart)
            result_proc = subprocess.Popen(args=args, preexec_fn=os.setpgrp)
            log.info("teuthology-results PID: %s", result_proc.pid)
        else:
            log.info('Creating archive dir %s', archive_path_full)
            safepath.makedirs(ctx.archive_dir, safe_archive)
            log.info('Running job %d', job.jid)
            run_job(job_config, teuth_bin_path)
        job.delete()

def run_with_watchdog(process, job_config):
    job_start_time = datetime.utcnow()

    # Only push the information that's relevant to the watchdog, to save db
    # load
    job_info = dict(
        name=job_config['name'],
        job_id=job_config['job_id'],
    )

    # Sleep once outside of the loop to avoid double-posting jobs
    time.sleep(teuth_config.watchdog_interval)
    symlink_worker_log(job_config['worker_log'], job_config['archive_path'])
    while process.poll() is None:
        # Kill jobs that have been running longer than the global max
        run_time = datetime.utcnow() - job_start_time
        total_seconds = run_time.days * 60 * 60 * 24 + run_time.seconds
        if total_seconds > teuth_config.max_job_time:
            log.warning("Job ran longer than {max}s. Killing...".format(
                max=teuth_config.max_job_time))
            kill_job(job_info['name'], job_info['job_id'],
                     teuth_config.archive_base)

        report.try_push_job_info(job_info, dict(status='running'))
        time.sleep(teuth_config.watchdog_interval)

    # The job finished. Let's make sure paddles knows.
    branches_sans_reporting = ('argonaut', 'bobtail', 'cuttlefish', 'dumpling')
    if job_config.get('teuthology_branch') in branches_sans_reporting:
        # The job ran with a teuthology branch that may not have the reporting
        # feature. Let's call teuthology-report (which will be from the master
        # branch) to report the job manually.
        cmd = "teuthology-report -v -D -r {run_name} -j {job_id}".format(
            run_name=job_info['name'],
            job_id=job_info['job_id'])
        try:
            log.info("Executing %s" % cmd)
            report_proc = subprocess.Popen(cmd, shell=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.STDOUT)
            while report_proc.poll() is None:
                for line in report_proc.stdout.readlines():
                    log.info(line.strip())
                time.sleep(1)
            log.info("Reported results via the teuthology-report command")
        except Exception:
            log.exception("teuthology-report failed")
    else:
        # Let's make sure that paddles knows the job is finished. We don't know
        # the status, but if it was a pass or fail it will have already been
        # reported to paddles. In that case paddles ignores the 'dead' status.
        # If the job was killed, paddles will use the 'dead' status.
        report.try_push_job_info(job_info, dict(status='dead'))

def run_job(job_config, teuth_bin_path):
    suite_path = job_config['suite_path']
    arg = [
        os.path.join(teuth_bin_path, 'teuthology'),
    ]
    # The following is for compatibility with older schedulers, from before we
    # started merging the contents of job_config['config'] into job_config
    # itself.
    if 'config' in job_config:
        inner_config = job_config.pop('config')
        if not isinstance(inner_config, dict):
            log.warn("run_job: job_config['config'] isn't a dict, it's a %s",
                     str(type(inner_config)))
        else:
            job_config.update(inner_config)

    if job_config['verbose']:
        arg.append('-v')

    arg.extend([
        '--lock',
        '--block',
        '--owner', job_config['owner'],
        '--archive', job_config['archive_path'],
        '--name', job_config['name'],
    ])
    if job_config['description'] is not None:
        arg.extend(['--description', job_config['description']])
    arg.append('--')

    with tempfile.NamedTemporaryFile(prefix='teuthology-worker.',
                                     suffix='.tmp',) as tmp:
        yaml.safe_dump(data=job_config, stream=tmp)
        tmp.flush()
        arg.append(tmp.name)
        env = os.environ.copy()
        python_path = env.get('PYTHONPATH', '')
        python_path = ':'.join([suite_path, python_path]).strip(':')
        env['PYTHONPATH'] = python_path
        p = subprocess.Popen(args=arg, env=env)
        log.info("Job archive: %s", job_config['archive_path'])
        log.info("Job PID: %s", str(p.pid))

        if teuth_config.results_server:
            log.info("Running with watchdog")
            try:
                run_with_watchdog(p, job_config)
            except Exception:
                log.exception("run_with_watchdog had an unhandled exception")
                raise
        else:
            log.info("Running without watchdog")
            # This sleep() is to give the child time to start up and create the
            # archive dir.
            time.sleep(5)
            symlink_worker_log(job_config['worker_log'],
                               job_config['archive_path'])
            p.wait()

        if p.returncode != 0:
            log.error('Child exited with code %d', p.returncode)
        else:
            log.info('Success!')

def symlink_worker_log(worker_log_path, archive_dir):
    try:
        log.debug("Worker log: %s", worker_log_path)
        os.symlink(worker_log_path, os.path.join(archive_dir, 'worker.log'))
    except Exception:
        log.exception("Failed to symlink worker log")
<|endoftext|>


<commit_before>from django.test import TestCase

from .text_viewer_tvof import TextViewerAPITvof
from django.http.request import HttpRequest

class TextViewerTests(TestCase):

    def test_request_documents(self):
        self._test_request()

    def _test_request_document_wrong(self):
        self._test_request('FRXYZ')

    def test_request_document(self):
        self._test_request('Fr20125')

    def test_request_section(self):
        self._test_request('Fr20125/interpretive/section/588')

    def test_request_whole(self):
        self._test_request('Fr20125/interpretive/whole/whole')

    def test_request_section_default(self):
        self._test_request('Fr20125/interpretive/section/default')

    def test_request_default_default(self):
        self._test_request('Fr20125/interpretive/default/default')

    def test_request_chunk_default_default_default(self):
        self._test_request('Fr20125/default/default/default')

    def _test_request(self, path=None, params=None):
        path = path or ''
        request = HttpRequest()

        tv = TextViewerAPITvof()
        tv.process_request(request, path)
        res = tv.get_response()

        # print res
        has_errors = bool(res.get('errors'))

        if has_errors:
            print repr(path)
            print repr(res.get('errors'))

        self.assertIs(has_errors, False)
<commit_msg>fixed text viewer tests were failing on travis because kiln is required.<commit_after>from django.test import TestCase
from django.conf import settings

from .text_viewer_tvof import TextViewerAPITvof
from django.http.request import HttpRequest

# only when TEST_TEXT_VIEWER = True in your local.py
# because kiln api not available to Travis
if getattr(settings, 'TEST_TEXT_VIEWER', False):
    class TextViewerTests(TestCase):

        def test_request_documents(self):
            self._test_request()

        def _test_request_document_wrong(self):
            self._test_request('FRXYZ')

        def test_request_document(self):
            self._test_request('Fr20125')

        def test_request_section(self):
            self._test_request('Fr20125/interpretive/section/588')

        def test_request_whole(self):
            self._test_request('Fr20125/interpretive/whole/whole')

        def test_request_section_default(self):
            self._test_request('Fr20125/interpretive/section/default')

        def test_request_default_default(self):
            self._test_request('Fr20125/interpretive/default/default')

        def test_request_chunk_default_default_default(self):
            self._test_request('Fr20125/default/default/default')

        def _test_request(self, path=None, params=None):
            path = path or ''
            request = HttpRequest()

            tv = TextViewerAPITvof()
            tv.process_request(request, path)
            res = tv.get_response()

            # print res
            has_errors = bool(res.get('errors'))

            if has_errors:
                print repr(path)
                print repr(res.get('errors'))

            self.assertIs(has_errors, False)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import subprocess

from docker_management import ContainerManager
import os

def start_servers(docker):
    # start db docker on unique port
    # load data into db from flat files in test case
    # start the dashboard server on unique port
    db_port = 15550  # TODO: make the db port configurable in data_access.py so it can be set on the command line
    db_container = _start_db_docker(docker, db_port)

    application_port = 4545
    gocd_dash_path = os.environ['gocd_dash']
    application_process = subprocess.Popen(["/usr/bin/env", "python3", gocd_dash_path, "-b", str(application_port)])

    return db_container, application_process

def stop_servers(docker, db, application):
    application.kill()
    docker.stop_container(db)

def perform_testcase():
    print("Starting test workflow for GO CD Dashboard")
    with open("gui_log.txt", "w") as log:
        with open("urls.txt") as urls:
            for url in urls:
                log.write("url: {}".format(url))
                # navigate to the url
                # make ascii art of the page
                # store the ascii art in the 'gui_log.txt' file
                sprocess_call = subprocess.Popen(["/usr/bin/lynx", "-dump", url], stdout=subprocess.PIPE)
                out = sprocess_call.communicate()[0]
                log.write(out)

def main():
    docker = ContainerManager(
        "dockerregistry.pagero.local")  # TODO: Put this image on docker hub instead of our internal docker registry
    db, application = start_servers(docker)
    try:
        import time
        time.sleep(2)
        perform_testcase()
    finally:
        stop_servers(docker, db, application)

def _start_db_docker(docker, dbport):
    db_image = "go-analysis-db"
    db_image_tag = "1.1.1-GO"

    return docker.start_db_container(db_image, db_image_tag, dbport)

if __name__ == "__main__":
    main()
<commit_msg>Database config for test server<commit_after>#!/usr/bin/env python3

import subprocess

from docker_management import ContainerManager
import os

def start_servers(docker):
    db_port = 15550
    db_container = _start_db_docker(docker, db_port)

    application_port = 4545
    gocd_dash_path = os.environ['gocd_dash']
    application_process = subprocess.Popen(["/usr/bin/env", "python3", gocd_dash_path, "-b", str(application_port), "--db-port", str(db_port)])

    return db_container, application_process

def stop_servers(docker, db, application):
    application.kill()
    docker.stop_container(db)

def perform_testcase():
    print("Starting test workflow for GO CD Dashboard")
    with open("gui_log.txt", "w") as log:
        with open("urls.txt") as urls:
            for url in urls:
                log.write("url: {}".format(url))
                # navigate to the url
                # make ascii art of the page
                # store the ascii art in the 'gui_log.txt' file
                sprocess_call = subprocess.Popen(["/usr/bin/lynx", "-dump", url], stdout=subprocess.PIPE)
                out = sprocess_call.communicate()[0]
                log.write(out)

def main():
    docker = ContainerManager(
        "dockerregistry.pagero.local")  # TODO: Put this image on docker hub instead of our internal docker registry
    db, application = start_servers(docker)
    try:
        import time
        time.sleep(2)
        perform_testcase()
    finally:
        stop_servers(docker, db, application)

def _start_db_docker(docker, dbport):
    db_image = "go-analysis-db"
    db_image_tag = "1.1.1-GO"

    return docker.start_db_container(db_image, db_image_tag, dbport)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># forth.py  25/12/2015  D.J.Whale
#
# An experiment to write a minimal FORTH language on top of Python.
# The main purpose of this is to study the design of the FORTH language
# by attempting a modern implementation of it.
#
# The idea is to allow multiple simultaneous Forth context objects,
# perhaps with some shared data with copy-on-write semantics,
# so that core objects could be written in little Forth package
# objects, and integrated into a bigger system (so each Forth
# context is like a mini self-contained sandbox with it's own
# memory space and execution thread).

# BOOT REST OF LANGUAGE (do this in FORTH itself)
#   other compiler words
#   conditionals
#   other forth DS words
#   other forth RS words
#   other forth words

#----- MEMORY -----------------------------------------------------------------
#
# Access to a block of memory, basically a Python list.

class Memory():
    def __init__(self, size=65535):
        self.size = size
        self.mem = [0 for i in range(size)]

    def write(self, addr, value):
        self.mem[addr] = value

    def read(self, addr):
        return self.mem[addr]

#---- INPUT -------------------------------------------------------------------
#
# Interface to keyboard input

class Input():
    def __init__(self):
        pass

    def check(self):
        return True

    def read(self):
        return '*'

#----- OUTPUT -----------------------------------------------------------------
#
# Interface to screen output

class Output():
    def __init__(self):
        pass

    def write(self, ch):
        print("out:%s" % ch)

#----- DISK -------------------------------------------------------------------
#
# TODO: Probably an interface to reading and writing blocks in a nominated
# disk file image.

class Disk():
    def __init__(self):
        pass

#----- PYTHON WRAPPERS FOR FORTH DATA STRUCTURES -----------------------------------
#
# A useful abstraction to allow Python to meddle directly with Forth's
# data structure regions in memory. This is useful when you want to rewrite
# the implementation code for a WORD in Python to get better execution speed.

class SysVars():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, create, read, write

class BlockBuffers():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, read, write, erase
    # cache index

class Dictionary():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, create, find, read, forget, here

class UserVars():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr. create, find, read, write

class ReturnStack():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, push, pop, top, clear, size

class TextInputBuffer():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, erase, read, write, advance, retard

class DataStack():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, push, pop, top, clear, size

class Pad():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, clear, read, write, advance, retard, reset, move?

#----- FORTH CONTEXT ----------------------------------------------------------
#
# The Forth language - knits everything together into one helpful object.

class Forth:
    mem    = Memory()
    input  = Input()
    output = Output()
    map    = []

    def region(self, name, spec):
        size  = spec[1]
        if size < 0:
            # grows down towards low memory
            start = spec[0] - -size
        else:
            # grows up towards high memory
            start = spec[0]
        ptr = spec[0]
        limit = start + size

        self.map.append((name, start, abs(size)))
        return start, ptr, limit

    def show_memmap(self):
        last_end = 0
        for i in self.map:
            name, start, size = i
            if start != last_end:
                uname  = "UNUSED"
                ustart = last_end
                uend   = start-1
                usize  = uend-ustart-1
                print("%10s %6d %6d %6d" %(uname, ustart, uend, usize))
            print("%10s %6d %6d %6d" % (name, start, start+size-1, size))
            last_end = start + size

    def boot(self):
        self.makeds()
        #self.boot_sys_vars() (including init of base/ptr vars of various regions)
        #self.boot_user_vars()
        #self.boot_native_words()
        #self.boot_forth_words()
        #self.boot_min_interpreter()
        #self.boot_min_compiler()
        #self.boot_min_editor()
        return self

    def makeds(self):
        # BOOT DATASTRUCTURES (base, ptr, limit for each)

        SV_MEM   = (0,               +1024      )
        EL_MEM   = (1024,            +0         )
        DICT_MEM = (1024,            +1024      )
        PAD_MEM  = (2048,            +80        )
        DS_MEM   = (8192,            -1024      ) # grows downwards
        TIB_MEM  = (8192,            +80        )
        RS_MEM   = (16384,           -1024      ) # grows downwards
        UV_MEM   = (16384,           +1024      )
        BB_MEM   = (65536-(1024*2),  +(1024*2)  )

        #   init sysvars
        svbase, svptr, svlimit = self.region("SV", SV_MEM)
        self.sv = SysVars(self.mem, svbase, svptr, svlimit)

        #   init elective space??
        #elbase, elptr, ellimit = self.region("EL", at=, size=)

        #   init dictionary
        dictbase, dictptr, dictlimit = self.region("DICT", DICT_MEM)
        self.dict = Dictionary(self.mem, dictbase, dictptr, dictlimit)

        #   init pad
        padbase, padptr, padlimit = self.region("PAD", PAD_MEM)
        self.pad = Pad(self.mem, padbase, padptr, padlimit)

        #   init data stack
        dsbase, dsptr, dslimit = self.region("DS", DS_MEM)
        self.ds = DataStack(self.mem, dsbase, dsptr, dslimit)

        #   init text input buffer
        tibbase, tibptr, tiblimit = self.region("TIB", TIB_MEM)
        self.tib = TextInputBuffer(self.mem, tibbase, tibptr, tiblimit)

        #   init return stack
        rsbase, rsptr, rslimit = self.region("RS", RS_MEM)
        self.rs = ReturnStack(self.mem, rsbase, rsptr, rslimit)

        #   init user variables (BASE, S0,...)
        uvbase, uvptr, uvlimit = self.region("UV", UV_MEM)
        self.uv = UserVars(self.mem, uvbase, uvptr, uvlimit)

        #   init block buffers
        bbbase, bbptr, bblimit = self.region("BB", BB_MEM)
        self.bb = BlockBuffers(self.mem, bbbase, bbptr, bblimit)

        self.show_memmap()

    def boot_min_interpreter(self):
        # BOOT MIN INTERPRETER (some in FORTH, some in Python?)
        #   BYE
        #   BL CR . ." EMIT
        #   RESET ABORT
        #   KEY? KEY
        #   FALSE TRUE BEGIN UNTIL
        #   WORD NUMBER
        #   EXPECT SPAN
        #   QUERY
        #   EXECUTE
        #   INTERPRET
        #   QUIT EXIT
        pass

    def boot_min_compiler(self):
        # BOOT MIN COMPILER (might do this in FORTH itself)
        #   , @ ,C VARIABLE
        #   CREATE
        #   <BUILDS DOES> COMPILE
        #   : ;
        pass

    def run(self):
        #NOTE: can boot() then clone(), and then customise and run() multiple clones
        # optionally load app?
        # run main interpreter loop (optionally in a thread?)
        # only gets here when see a 'BYE' command.
        print("warning: No interpreter yet")

#----- RUNNER -----------------------------------------------------------------

def test():
    f = Forth().boot()
    f.run()

if __name__ == "__main__":
    test()

# END
<commit_msg>Now checks and rejects overlapping memory regions<commit_after># forth.py  25/12/2015  D.J.Whale
#
# An experiment to write a minimal FORTH language on top of Python.
# The main purpose of this is to study the design of the FORTH language
# by attempting a modern implementation of it.
#
# The idea is to allow multiple simultaneous Forth context objects,
# perhaps with some shared data with copy-on-write semantics,
# so that core objects could be written in little Forth package
# objects, and integrated into a bigger system (so each Forth
# context is like a mini self-contained sandbox with it's own
# memory space and execution thread).

# BOOT REST OF LANGUAGE (do this in FORTH itself)
#   other compiler words
#   conditionals
#   other forth DS words
#   other forth RS words
#   other forth words

#----- MEMORY -----------------------------------------------------------------
#
# Access to a block of memory, basically a Python list.

class Memory():
    def __init__(self, size=65535):
        self.size = size
        self.mem = [0 for i in range(size)]

    def write(self, addr, value):
        self.mem[addr] = value

    def read(self, addr):
        return self.mem[addr]

#---- INPUT -------------------------------------------------------------------
#
# Interface to keyboard input

class Input():
    def __init__(self):
        pass

    def check(self):
        return True

    def read(self):
        return '*'

#----- OUTPUT -----------------------------------------------------------------
#
# Interface to screen output

class Output():
    def __init__(self):
        pass

    def write(self, ch):
        print("out:%s" % ch)

#----- DISK -------------------------------------------------------------------
#
# TODO: Probably an interface to reading and writing blocks in a nominated
# disk file image.

class Disk():
    def __init__(self):
        pass

#----- PYTHON WRAPPERS FOR FORTH DATA STRUCTURES -----------------------------------
#
# A useful abstraction to allow Python to meddle directly with Forth's
# data structure regions in memory. This is useful when you want to rewrite
# the implementation code for a WORD in Python to get better execution speed.

class SysVars():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, create, read, write

class BlockBuffers():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, read, write, erase
    # cache index

class Dictionary():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, create, find, read, forget, here

class UserVars():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr. create, find, read, write

class ReturnStack():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, push, pop, top, clear, size

class TextInputBuffer():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, erase, read, write, advance, retard

class DataStack():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, push, pop, top, clear, size

class Pad():
    def __init__(self, mem, base, ptr, limit):
        pass
    # addr, clear, read, write, advance, retard, reset, move?

#----- FORTH CONTEXT ----------------------------------------------------------
#
# The Forth language - knits everything together into one helpful object.

class Forth:
    mem    = Memory()
    input  = Input()
    output = Output()
    map    = []

    def region(self, name, spec):
        size  = spec[1]
        if size < 0:
            # grows down towards low memory
            start = spec[0] - -size
        else:
            # grows up towards high memory
            start = spec[0]
        ptr = spec[0]
        end = start + size

        # check for overlaps with an existing region
        for i in self.map:
            iname, istart, isize = i
            iend = istart + isize-1
            if (start >= istart and start <= iend) or (end >= istart and end <= iend):
                raise ValueError("Region %s overlaps with %s" % (name, iname))

        self.map.append((name, start, abs(size)))
        return start, ptr, end

    def show_memmap(self):
        last_end = 0
        for i in self.map:
            name, start, size = i
            if start != last_end:
                uname  = "UNUSED"
                ustart = last_end
                uend   = start-1
                usize  = uend-ustart-1
                print("%10s %6d %6d %6d" %(uname, ustart, uend, usize))
            print("%10s %6d %6d %6d" % (name, start, start+size-1, size))
            last_end = start + size

    def boot(self):
        self.makeds()
        #self.boot_sys_vars() (including init of base/ptr vars of various regions)
        #self.boot_user_vars()
        #self.boot_native_words()
        #self.boot_forth_words()
        #self.boot_min_interpreter()
        #self.boot_min_compiler()
        #self.boot_min_editor()
        return self

    def makeds(self):
        # BOOT DATASTRUCTURES (base, ptr, limit for each)

        SV_MEM   = (0,               +1024      )
        EL_MEM   = (1024,            +0         )
        DICT_MEM = (1024,            +1024      )
        PAD_MEM  = (2048,            +80        )
        DS_MEM   = (8192,            -1024      ) # grows downwards
        TIB_MEM  = (8192,            +80        )
        RS_MEM   = (16384,           -1024      ) # grows downwards
        UV_MEM   = (16384,           +1024      )
        BB_MEM   = (65536-(1024*2),  +(1024*2)  )

        #   init sysvars
        svbase, svptr, svlimit = self.region("SV", SV_MEM)
        self.sv = SysVars(self.mem, svbase, svptr, svlimit)

        #   init elective space??
        #elbase, elptr, ellimit = self.region("EL", at=, size=)

        #   init dictionary
        dictbase, dictptr, dictlimit = self.region("DICT", DICT_MEM)
        self.dict = Dictionary(self.mem, dictbase, dictptr, dictlimit)

        #   init pad
        padbase, padptr, padlimit = self.region("PAD", PAD_MEM)
        self.pad = Pad(self.mem, padbase, padptr, padlimit)

        #   init data stack
        dsbase, dsptr, dslimit = self.region("DS", DS_MEM)
        self.ds = DataStack(self.mem, dsbase, dsptr, dslimit)

        #   init text input buffer
        tibbase, tibptr, tiblimit = self.region("TIB", TIB_MEM)
        self.tib = TextInputBuffer(self.mem, tibbase, tibptr, tiblimit)

        #   init return stack
        rsbase, rsptr, rslimit = self.region("RS", RS_MEM)
        self.rs = ReturnStack(self.mem, rsbase, rsptr, rslimit)

        #   init user variables (BASE, S0,...)
        uvbase, uvptr, uvlimit = self.region("UV", UV_MEM)
        self.uv = UserVars(self.mem, uvbase, uvptr, uvlimit)

        #   init block buffers
        bbbase, bbptr, bblimit = self.region("BB", BB_MEM)
        self.bb = BlockBuffers(self.mem, bbbase, bbptr, bblimit)

        self.show_memmap()

    def boot_min_interpreter(self):
        # BOOT MIN INTERPRETER (some in FORTH, some in Python?)
        #   BYE
        #   BL CR . ." EMIT
        #   RESET ABORT
        #   KEY? KEY
        #   FALSE TRUE BEGIN UNTIL
        #   WORD NUMBER
        #   EXPECT SPAN
        #   QUERY
        #   EXECUTE
        #   INTERPRET
        #   QUIT EXIT
        pass

    def boot_min_compiler(self):
        # BOOT MIN COMPILER (might do this in FORTH itself)
        #   , @ ,C VARIABLE
        #   CREATE
        #   <BUILDS DOES> COMPILE
        #   : ;
        pass

    def run(self):
        #NOTE: can boot() then clone(), and then customise and run() multiple clones
        # optionally load app?
        # run main interpreter loop (optionally in a thread?)
        # only gets here when see a 'BYE' command.
        print("warning: No interpreter yet")

#----- RUNNER -----------------------------------------------------------------

def test():
    f = Forth().boot()
    f.run()

if __name__ == "__main__":
    test()

# END
<|endoftext|>


<commit_before>import itertools
import json
import logging
import os
import re
import six
import django

from django.utils import timezone, _os, translation
from django.utils.encoding import force_str

from tg_react import settings

def get_plural_forms(locale):
    plural_forms_re = re.compile(r'^"Plural-Forms: (?P<value>.+?)\\n"\s*$', re.MULTILINE | re.DOTALL)
    django_dir = os.path.normpath(os.path.join(os.path.dirname(_os.upath(django.__file__))))

    django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', 'django.po')
    if os.path.exists(django_po):
        with open(django_po, 'r', encoding='utf-8') as fp:
            m = plural_forms_re.search(fp.read())
        if m:
            return force_str(m.group('value')).strip().replace('"\n"', '')

    return 'nplurals=2; plural=(n != 1);'

def get_messages():
    try:
        from tg_react_tmp_i18n import all_messages

        result = all_messages()

    except ImportError:
        logging.warning('Did not find `tg_react_tmp_i18n.py` have you executed makemessages/compilemessages?')
        result = {}

    return result

def make_header(locale):
    now = timezone.now().isoformat()

    return {
        "po-revision-date": now,
        "mime-version": "1.0",
        "last-translator": "Automatic <hi@thorgate.eu>",
        "x-generator": "Python",
        "language": locale,
        "lang": locale,
        "content-transfer-encoding": "8bit",
        "project-id-version": "1.0.0",
        "pot-creation-date": now,
        "domain": "django",
        "report-msgid-bugs-to": "",
        "content-type": "text/plain; charset=UTF-8",
        "plural-forms": get_plural_forms(locale),
        "language-team": "Automatic <hi@thorgate.eu>"
    }

def collect_translations():
    languages = {}
    locale_data = {}
    js_messages = get_messages()

    for language_code, label in settings.LANGUAGES:
        languages[language_code] = '%s' % label

        # Add the meta object
        locale_data[language_code] = {}
        locale_data[language_code][""] = make_header(language_code)

        with translation.override(language_code):
            # Next code is largely taken from Django 1.10 djangojs logic @ django.views.i18n get_javascript_catalog
            pdict = {}
            maxcnts = {}
            trans = translation._trans.catalog()
            trans_cat = trans._catalog
            trans_fallback_cat = trans._fallback._catalog if trans._fallback else {}

            for key, value in itertools.chain(six.iteritems(trans_cat), six.iteritems(trans_fallback_cat)):
                # We only need to add string once into locale_data, this is why we check if key or msgid already
                # exists in the locale_data. Not doing this may result in incorrect translation strings from other
                # languages overwriting the correct ones.
                if isinstance(key, six.string_types):
                    if key not in js_messages.keys() or key in locale_data[language_code]:
                        continue
                    locale_data[language_code][key] = [value]
                elif isinstance(key, tuple):
                    msgid = key[0]
                    if msgid not in js_messages.keys():
                        continue
                    cnt = key[1]
                    maxcnts[msgid] = max(cnt, maxcnts.get(msgid, 0))
                    # In case of plurals the previously described logic does not work. Then we need to
                    # check if plural form already exists in translations.
                    if msgid not in pdict:
                        pdict.setdefault(msgid, {})[cnt] = value
                    elif cnt not in pdict[msgid]:
                        pdict.setdefault(msgid, {})[cnt] = value
                else:
                    raise TypeError(key)

            for k, v in pdict.items():
                locale_data[language_code][k] = [v.get(i, '') for i in range(maxcnts[k] + 1)]

    for key, value in locale_data.items():
        locale_data[key] = json.dumps(value)

    return languages, locale_data

def constants(context):
    all_languages, locale_data = collect_translations()

    return {
        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
        'LANGUAGE_COOKIE_NAME': settings.LANGUAGE_COOKIE_NAME,
        'LANGUAGES': all_languages,
        'LOCALE_DATA': locale_data,
    }
<commit_msg>Apparently the previous import was not working for the others too<commit_after>import itertools
import json
import logging
import os
import re
import six
import django

from django.utils import timezone, _os, translation
from django.utils.encoding import force_str

from django.conf import settings

def get_plural_forms(locale):
    plural_forms_re = re.compile(r'^"Plural-Forms: (?P<value>.+?)\\n"\s*$', re.MULTILINE | re.DOTALL)
    django_dir = os.path.normpath(os.path.join(os.path.dirname(_os.upath(django.__file__))))

    django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', 'django.po')
    if os.path.exists(django_po):
        with open(django_po, 'r', encoding='utf-8') as fp:
            m = plural_forms_re.search(fp.read())
        if m:
            return force_str(m.group('value')).strip().replace('"\n"', '')

    return 'nplurals=2; plural=(n != 1);'

def get_messages():
    try:
        from tg_react_tmp_i18n import all_messages

        result = all_messages()

    except ImportError:
        logging.warning('Did not find `tg_react_tmp_i18n.py` have you executed makemessages/compilemessages?')
        result = {}

    return result

def make_header(locale):
    now = timezone.now().isoformat()

    return {
        "po-revision-date": now,
        "mime-version": "1.0",
        "last-translator": "Automatic <hi@thorgate.eu>",
        "x-generator": "Python",
        "language": locale,
        "lang": locale,
        "content-transfer-encoding": "8bit",
        "project-id-version": "1.0.0",
        "pot-creation-date": now,
        "domain": "django",
        "report-msgid-bugs-to": "",
        "content-type": "text/plain; charset=UTF-8",
        "plural-forms": get_plural_forms(locale),
        "language-team": "Automatic <hi@thorgate.eu>"
    }

def collect_translations():
    languages = {}
    locale_data = {}
    js_messages = get_messages()

    for language_code, label in settings.LANGUAGES:
        languages[language_code] = '%s' % label

        # Add the meta object
        locale_data[language_code] = {}
        locale_data[language_code][""] = make_header(language_code)

        with translation.override(language_code):
            # Next code is largely taken from Django 1.10 djangojs logic @ django.views.i18n get_javascript_catalog
            pdict = {}
            maxcnts = {}
            trans = translation._trans.catalog()
            trans_cat = trans._catalog
            trans_fallback_cat = trans._fallback._catalog if trans._fallback else {}

            for key, value in itertools.chain(six.iteritems(trans_cat), six.iteritems(trans_fallback_cat)):
                # We only need to add string once into locale_data, this is why we check if key or msgid already
                # exists in the locale_data. Not doing this may result in incorrect translation strings from other
                # languages overwriting the correct ones.
                if isinstance(key, six.string_types):
                    if key not in js_messages.keys() or key in locale_data[language_code]:
                        continue
                    locale_data[language_code][key] = [value]
                elif isinstance(key, tuple):
                    msgid = key[0]
                    if msgid not in js_messages.keys():
                        continue
                    cnt = key[1]
                    maxcnts[msgid] = max(cnt, maxcnts.get(msgid, 0))
                    # In case of plurals the previously described logic does not work. Then we need to
                    # check if plural form already exists in translations.
                    if msgid not in pdict:
                        pdict.setdefault(msgid, {})[cnt] = value
                    elif cnt not in pdict[msgid]:
                        pdict.setdefault(msgid, {})[cnt] = value
                else:
                    raise TypeError(key)

            for k, v in pdict.items():
                locale_data[language_code][k] = [v.get(i, '') for i in range(maxcnts[k] + 1)]

    for key, value in locale_data.items():
        locale_data[key] = json.dumps(value)

    return languages, locale_data

def constants(context):
    all_languages, locale_data = collect_translations()

    return {
        'LANGUAGE_CODE': settings.LANGUAGE_CODE,
        'LANGUAGE_COOKIE_NAME': settings.LANGUAGE_COOKIE_NAME,
        'LANGUAGES': all_languages,
        'LOCALE_DATA': locale_data,
    }
<|endoftext|>


<commit_before># -*- coding: ISO-8859-15 -*-
# =============================================================================
# Copyright (c) 2004, 2006 Sean C. Gillies
# Copyright (c) 2007 STFC <http://www.stfc.ac.uk>
#
# Authors :
#          Dominic Lowe <d.lowe@rl.ac.uk>
#
# Contact email: d.lowe@rl.ac.uk
# =============================================================================

from urllib import urlencode
from urllib2 import urlopen
from owslib.etree import etree
import cgi
from StringIO import StringIO

#!/usr/bin/env python

import logging

class WCSBase(object):
    """Base class to be subclassed by version dependent WCS classes. Provides 'high-level' version independent methods"""
    def __new__(self,url, xml):
        """ overridden __new__ method

        @type url: string
        @param url: url of WCS capabilities document
        @type xml: string
        @param xml: elementtree object
        @return: inititalised WCSBase object
        """
        obj=object.__new__(self, url, xml)
        obj.__init__(url, xml)
        self.log = logging.getLogger()
        consoleh  = logging.StreamHandler()
        self.log.addHandler(consoleh)
        return obj

    def __init__(self):
        pass

    def getDescribeCoverage(self,identifier):
         reader = DescribeCoverageReader(self.version, identifier)
         #Note: should implement some sort of cache so the same request isn't repeated
         return reader.read(self.url)

    def setLogLevel(self, level='CRITICAL'):
        #accepts level = DEBUG, INFO, WARNING, ERROR, CRITICAL
        if level=='DEBUG':
            self.log.setLevel(logging.DEBUG)
        elif level=='INFO':
            self.log.setLevel(logging.INFO)
        elif level=='WARNING':
            self.log.setLevel(logging.WARNING)
        elif level=='ERROR':
            self.log.setLevel(logging.ERROR)
        elif level=='CRITICAL':
            self.log.setLevel(logging.CRITICAL)

class WCSCapabilitiesReader(object):
    """Read and parses WCS capabilities document into a lxml.etree infoset
    """

    def __init__(self, version):
        """Initialize
        @type version: string
        @param version: WCS Version parameter e.g '1.0.0'
        """
        self.version = version
        self._infoset = None

    def capabilities_url(self, service_url):
        """Return a capabilities url
        @type service_url: string
        @param service_url: base url of WCS service
        @rtype: string
        @return: getCapabilities URL
        """
        qs = []
        if service_url.find('?') != -1:
            qs = cgi.parse_qsl(service_url.split('?')[1])

        params = [x[0] for x in qs]

        if 'service' not in params:
            qs.append(('service', 'WCS'))
        if 'request' not in params:
            qs.append(('request', 'GetCapabilities'))
        if 'version' not in params:
            qs.append(('version', self.version))

        urlqs = urlencode(tuple(qs))
        return service_url.split('?')[0] + '?' + urlqs

    def read(self, service_url):
        """Get and parse a WCS capabilities document, returning an
        elementtree tree

        @type service_url: string
        @param service_url: The base url, to which is appended the service,
        version, and request parameters
        @rtype: elementtree tree
        @return: An elementtree tree representation of the capabilities document
        """
        request = self.capabilities_url(service_url)
        u = urlopen(request)
        return etree.fromstring(u.read())

    def readString(self, st):
        """Parse a WCS capabilities document, returning an
        instance of WCSCapabilitiesInfoset

        string should be an XML capabilities document
        """
        if not isinstance(st, str):
            raise ValueError("String must be of type string, not %s" % type(st))
        return etree.fromstring(st)

class DescribeCoverageReader(object):
    """Read and parses WCS DescribeCoverage document into a lxml.etree infoset
    """

    def __init__(self, version, identifier):
        """Initialize
        @type version: string
        @param version: WCS Version parameter e.g '1.0.0'
        """
        self.version = version
        self._infoset = None
        self.identifier=identifier

    def descCov_url(self, service_url):
        """Return a describe coverage url
        @type service_url: string
        @param service_url: base url of WCS service
        @rtype: string
        @return: getCapabilities URL
        """
        qs = []
        if service_url.find('?') != -1:
            qs = cgi.parse_qsl(service_url.split('?')[1])

        params = [x[0] for x in qs]

        if 'service' not in params:
            qs.append(('service', 'WCS'))
        if 'request' not in params:
            qs.append(('request', 'DescribeCoverage'))
        if 'version' not in params:
            qs.append(('version', self.version))
        if self.version == '1.0.0':
            if 'coverage' not in params:
                qs.append(('coverage', self.identifier))
        elif self.version == '1.1.0':
            #NOTE: WCS 1.1.0 is ambigous about whether it should be identifier
            #or identifiers (see tables 9, 10 of specification)
            if 'identifiers' not in params:
                qs.append(('identifiers', self.identifier))
            if 'identifier' not in params:
                qs.append(('identifier', self.identifier))
                qs.append(('format', 'text/xml'))
        urlqs = urlencode(tuple(qs))
        return service_url.split('?')[0] + '?' + urlqs

    def read(self, service_url):
        """Get and parse a Describe Coverage document, returning an
        elementtree tree

        @type service_url: string
        @param service_url: The base url, to which is appended the service,
        version, and request parameters
        @rtype: elementtree tree
        @return: An elementtree tree representation of the capabilities document
        """
        request = self.descCov_url(service_url)
        u = urlopen(request)
        return etree.fromstring(u.read())

<commit_msg>adding caching mechanism to getDescribeCoverage method - similar to that suggested by Goyo<commit_after># -*- coding: ISO-8859-15 -*-
# =============================================================================
# Copyright (c) 2004, 2006 Sean C. Gillies
# Copyright (c) 2007 STFC <http://www.stfc.ac.uk>
#
# Authors :
#          Dominic Lowe <d.lowe@rl.ac.uk>
#
# Contact email: d.lowe@rl.ac.uk
# =============================================================================

from urllib import urlencode
from urllib2 import urlopen
from owslib.etree import etree
import cgi
from StringIO import StringIO

#!/usr/bin/env python

import logging

class WCSBase(object):
    """Base class to be subclassed by version dependent WCS classes. Provides 'high-level' version independent methods"""
    def __new__(self,url, xml):
        """ overridden __new__ method

        @type url: string
        @param url: url of WCS capabilities document
        @type xml: string
        @param xml: elementtree object
        @return: inititalised WCSBase object
        """
        obj=object.__new__(self, url, xml)
        obj.__init__(url, xml)
        self.log = logging.getLogger()
        consoleh  = logging.StreamHandler()
        self.log.addHandler(consoleh)
        self._describeCoverage = {} #cache for DescribeCoverage responses
        return obj

    def __init__(self):
        pass

    def getDescribeCoverage(self, identifier):
        ''' returns a describe coverage document - checks the internal cache to see if it has been fetched before '''
        if identifier not in self._describeCoverage.keys():
            reader = DescribeCoverageReader(self.version, identifier)
            self._describeCoverage[identifier] = reader.read(self.url)
        return self._describeCoverage[identifier]

    def setLogLevel(self, level='CRITICAL'):
        #accepts level = DEBUG, INFO, WARNING, ERROR, CRITICAL
        if level=='DEBUG':
            self.log.setLevel(logging.DEBUG)
        elif level=='INFO':
            self.log.setLevel(logging.INFO)
        elif level=='WARNING':
            self.log.setLevel(logging.WARNING)
        elif level=='ERROR':
            self.log.setLevel(logging.ERROR)
        elif level=='CRITICAL':
            self.log.setLevel(logging.CRITICAL)

class WCSCapabilitiesReader(object):
    """Read and parses WCS capabilities document into a lxml.etree infoset
    """

    def __init__(self, version):
        """Initialize
        @type version: string
        @param version: WCS Version parameter e.g '1.0.0'
        """
        self.version = version
        self._infoset = None

    def capabilities_url(self, service_url):
        """Return a capabilities url
        @type service_url: string
        @param service_url: base url of WCS service
        @rtype: string
        @return: getCapabilities URL
        """
        qs = []
        if service_url.find('?') != -1:
            qs = cgi.parse_qsl(service_url.split('?')[1])

        params = [x[0] for x in qs]

        if 'service' not in params:
            qs.append(('service', 'WCS'))
        if 'request' not in params:
            qs.append(('request', 'GetCapabilities'))
        if 'version' not in params:
            qs.append(('version', self.version))

        urlqs = urlencode(tuple(qs))
        return service_url.split('?')[0] + '?' + urlqs

    def read(self, service_url):
        """Get and parse a WCS capabilities document, returning an
        elementtree tree

        @type service_url: string
        @param service_url: The base url, to which is appended the service,
        version, and request parameters
        @rtype: elementtree tree
        @return: An elementtree tree representation of the capabilities document
        """
        request = self.capabilities_url(service_url)
        u = urlopen(request)
        return etree.fromstring(u.read())

    def readString(self, st):
        """Parse a WCS capabilities document, returning an
        instance of WCSCapabilitiesInfoset

        string should be an XML capabilities document
        """
        if not isinstance(st, str):
            raise ValueError("String must be of type string, not %s" % type(st))
        return etree.fromstring(st)

class DescribeCoverageReader(object):
    """Read and parses WCS DescribeCoverage document into a lxml.etree infoset
    """

    def __init__(self, version, identifier):
        """Initialize
        @type version: string
        @param version: WCS Version parameter e.g '1.0.0'
        """
        self.version = version
        self._infoset = None
        self.identifier=identifier

    def descCov_url(self, service_url):
        """Return a describe coverage url
        @type service_url: string
        @param service_url: base url of WCS service
        @rtype: string
        @return: getCapabilities URL
        """
        qs = []
        if service_url.find('?') != -1:
            qs = cgi.parse_qsl(service_url.split('?')[1])

        params = [x[0] for x in qs]

        if 'service' not in params:
            qs.append(('service', 'WCS'))
        if 'request' not in params:
            qs.append(('request', 'DescribeCoverage'))
        if 'version' not in params:
            qs.append(('version', self.version))
        if self.version == '1.0.0':
            if 'coverage' not in params:
                qs.append(('coverage', self.identifier))
        elif self.version == '1.1.0':
            #NOTE: WCS 1.1.0 is ambigous about whether it should be identifier
            #or identifiers (see tables 9, 10 of specification)
            if 'identifiers' not in params:
                qs.append(('identifiers', self.identifier))
            if 'identifier' not in params:
                qs.append(('identifier', self.identifier))
                qs.append(('format', 'text/xml'))
        urlqs = urlencode(tuple(qs))
        return service_url.split('?')[0] + '?' + urlqs

    def read(self, service_url):
        """Get and parse a Describe Coverage document, returning an
        elementtree tree

        @type service_url: string
        @param service_url: The base url, to which is appended the service,
        version, and request parameters
        @rtype: elementtree tree
        @return: An elementtree tree representation of the capabilities document
        """
        request = self.descCov_url(service_url)
        u = urlopen(request)
        return etree.fromstring(u.read())

<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals
import numpy
from preshed.maps import PreshMap
from pathlib import Path

try:
    import cupy
    from cupy import get_array_module
except ImportError:
    cupy = None
    get_array_module = lambda _: numpy

try:
    basestring
except NameError:
    basestring = str

def get_ops(ops):
    from .ops import NumpyOps, CupyOps
    if ops in ('numpy', 'cpu') or (isinstance(ops, int) and ops < 0):
        return NumpyOps
    elif ops in ('cupy', 'gpu') or (isinstance(ops, int) and ops >= 0):
        return CupyOps
    else:
        raise ValueError("Invalid ops (or device) description: %s" % ops)

def mark_sentence_boundaries(sequences, drop=0.): # pragma: no cover
    '''Pad sentence sequences with EOL markers.'''
    for sequence in sequences:
        sequence.insert(0, '-EOL-')
        sequence.insert(0, '-EOL-')
        sequence.append('-EOL-')
        sequence.append('-EOL-')
    return sequences, None

def remap_ids(ops):
    id_map = {0: 0}
    def begin_update(ids, drop=0.):
        n_vector = len(id_map)
        for i, id_ in enumerate(ids):
            if id_ not in id_map:
                id_map[id_] = n_vector
                n_vector += 1
            ids[i] = id_map[id_]
        return ids, None
    return begin_update

def copy_array(dst, src, casting='same_kind', where=None):
    if isinstance(dst, numpy.ndarray) and isinstance(src, numpy.ndarray):
        dst[:] = src
    elif isinstance(dst, cupy.ndarray):
        src = cupy.array(src, copy=False)
        cupy.copyto(dst, src)
    else:
        numpy.copyto(dst, src)

def ensure_path(path):
    if isinstance(path, unicode) or isinstance(path, str):
        return Path(path)
    else:
        return path

def to_categorical(y, nb_classes=None):
    # From keras
    xp = get_array_module(y)
    if xp is cupy:
        y = y.get()
    y = numpy.array(y, dtype='int').ravel()
    if not nb_classes:
        nb_classes = numpy.max(y) + 1
    n = y.shape[0]
    categorical = numpy.zeros((n, nb_classes), dtype='float32')
    categorical[numpy.arange(n), y] = 1
    return xp.asarray(categorical)

def flatten_sequences(sequences, drop=0.): # pragma: no cover
    xp = get_array_module(sequences[0])
    return xp.concatenate(sequences), None

def partition(examples, split_size): # pragma: no cover
    examples = list(examples)
    numpy.random.shuffle(examples)
    n_docs = len(examples)
    split = int(n_docs * split_size)
    return examples[:split], examples[split:]

def minibatch(stream, batch_size=1000): # pragma: no cover
    if hasattr(stream, '__len__') and hasattr(stream, '__getitem__'):
        i = 0
        while i < len(stream):
            yield stream[i : i + batch_size]
            i += batch_size
    else:
        batch = []
        for X in stream:
            batch.append(X)
            if len(batch) >= batch_size:
                yield batch
                batch = []
        if len(batch) != 0:
            yield batch
<commit_msg>Fix Py2/3 code in thinc<commit_after>from __future__ import print_function, unicode_literals
import numpy
from preshed.maps import PreshMap
from pathlib import Path

try:
    import cupy
    from cupy import get_array_module
except ImportError:
    cupy = None
    get_array_module = lambda _: numpy

try:
    basestring
except NameError:
    basestring = str

try:
    unicode
except NameError:
    unicode = str

def get_ops(ops):
    from .ops import NumpyOps, CupyOps
    if ops in ('numpy', 'cpu') or (isinstance(ops, int) and ops < 0):
        return NumpyOps
    elif ops in ('cupy', 'gpu') or (isinstance(ops, int) and ops >= 0):
        return CupyOps
    else:
        raise ValueError("Invalid ops (or device) description: %s" % ops)

def mark_sentence_boundaries(sequences, drop=0.): # pragma: no cover
    '''Pad sentence sequences with EOL markers.'''
    for sequence in sequences:
        sequence.insert(0, '-EOL-')
        sequence.insert(0, '-EOL-')
        sequence.append('-EOL-')
        sequence.append('-EOL-')
    return sequences, None

def remap_ids(ops):
    id_map = {0: 0}
    def begin_update(ids, drop=0.):
        n_vector = len(id_map)
        for i, id_ in enumerate(ids):
            if id_ not in id_map:
                id_map[id_] = n_vector
                n_vector += 1
            ids[i] = id_map[id_]
        return ids, None
    return begin_update

def copy_array(dst, src, casting='same_kind', where=None):
    if isinstance(dst, numpy.ndarray) and isinstance(src, numpy.ndarray):
        dst[:] = src
    elif isinstance(dst, cupy.ndarray):
        src = cupy.array(src, copy=False)
        cupy.copyto(dst, src)
    else:
        numpy.copyto(dst, src)

def ensure_path(path):
    if isinstance(path, unicode) or isinstance(path, str):
        return Path(path)
    else:
        return path

def to_categorical(y, nb_classes=None):
    # From keras
    xp = get_array_module(y)
    if xp is cupy:
        y = y.get()
    y = numpy.array(y, dtype='int').ravel()
    if not nb_classes:
        nb_classes = numpy.max(y) + 1
    n = y.shape[0]
    categorical = numpy.zeros((n, nb_classes), dtype='float32')
    categorical[numpy.arange(n), y] = 1
    return xp.asarray(categorical)

def flatten_sequences(sequences, drop=0.): # pragma: no cover
    xp = get_array_module(sequences[0])
    return xp.concatenate(sequences), None

def partition(examples, split_size): # pragma: no cover
    examples = list(examples)
    numpy.random.shuffle(examples)
    n_docs = len(examples)
    split = int(n_docs * split_size)
    return examples[:split], examples[split:]

def minibatch(stream, batch_size=1000): # pragma: no cover
    if hasattr(stream, '__len__') and hasattr(stream, '__getitem__'):
        i = 0
        while i < len(stream):
            yield stream[i : i + batch_size]
            i += batch_size
    else:
        batch = []
        for X in stream:
            batch.append(X)
            if len(batch) >= batch_size:
                yield batch
                batch = []
        if len(batch) != 0:
            yield batch
<|endoftext|>