<commit_before># Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generates a SQLite DB containing test data downloaded from GCS."""

import argparse
import logging
import json
import os
import random
import re
import signal
import sys
import time
import urllib.parse
from xml.etree import cElementTree as ET

import multiprocessing
import multiprocessing.pool
import requests
import ruamel.yaml as yaml

import model

def pad_numbers(string):
    """Modify a string to make its numbers suitable for natural sorting."""
    return re.sub(r'\d+', lambda m: m.group(0).rjust(16, '0'), string)

WORKER_CLIENT = None  # used for multiprocessing

class GCSClient:
    def __init__(self, jobs_dir, metadata=None):
        self.jobs_dir = jobs_dir
        self.metadata = metadata or {}
        self.session = requests.Session()

    def _request(self, path, params, as_json=True):
        """GETs a JSON resource from GCS, with retries on failure.

        Retries are based on guidance from
        cloud.google.com/storage/docs/gsutil/addlhelp/RetryHandlingStrategy

        """
        url = f'https://www.googleapis.com/storage/v1/b/{path}'
        for retry in range(23):
            try:
                resp = self.session.get(url, params=params, stream=False)
                if 400 <= resp.status_code < 500 and resp.status_code != 429:
                    return None
                resp.raise_for_status()
                if as_json:
                    try:
                        return resp.json()
                    except json.decoder.JSONDecodeError:
                        logging.error('Failed to decode request for %s',
                                      urllib.parse.unquote(url))
                        return None
                return resp.text
            except requests.exceptions.RequestException:
                logging.exception('request failed %s', url)
            time.sleep(random.random() * min(60, 2 ** retry))

    @staticmethod
    def _parse_uri(path):
        if not path.startswith('gs://'):
            raise ValueError("Bad path: %s, should start with 'gs://'" % path)
        bucket, prefix = path[5:].split('/', 1)
        return bucket, prefix

    def get(self, path, as_json=False):
        """Get an object from GCS."""
        bucket, prefix = self._parse_uri(path)
        return self._request(f'{bucket}/o/{urllib.parse.quote(prefix, "")}',
                             {'alt': 'media'}, as_json=as_json)

    def ls(self,
           path,
           dirs=True,
           files=True,
           delim=True,
           item_field='name',
           build_limit=sys.maxsize,):
        """Lists objects under a path on gcs."""
        # pylint: disable=invalid-name

        bucket, prefix = self._parse_uri(path)
        params = {'prefix': prefix, 'fields': 'nextPageToken'}
        if delim:
            params['delimiter'] = '/'
            if dirs:
                params['fields'] += ',prefixes'
        if files:
            params['fields'] += f',items({item_field})'
        while build_limit > 0:
            resp = self._request(f'{bucket}/o', params)
            if resp is None:  # nothing under path?
                return
            for prefix in resp.get('prefixes', []):
                build_limit -= 1
                yield f'gs://{bucket}/{prefix}'
            for item in resp.get('items', []):
                if item_field == 'name':
                    build_limit -= 1
                    yield f'gs://{bucket}/{item["name"]}'
                else:
                    build_limit -= 1
                    yield item[item_field]
            if 'nextPageToken' not in resp:
                break
            params['pageToken'] = resp['nextPageToken']

    def ls_dirs(self, path):
        return self.ls(path, dirs=True, files=False)

    def _ls_junit_paths(self, build_dir):
        """Lists the paths of JUnit XML files for a build."""
        url = f'{build_dir}artifacts/'
        for path in self.ls(url):
            if re.match(r'.*/junit.*\.xml$', path):
                yield path

    def get_junits_from_build(self, build_dir):
        """Generates all tests for a build."""
        files = {}
        assert not build_dir.endswith('/')
        for junit_path in self._ls_junit_paths(build_dir + '/'):
            junit = self.get(junit_path)
            if junit is None:
                continue
            files[junit_path] = junit
        return files

    def _get_jobs(self):
        """Generates all jobs in the bucket."""
        for job_path in self.ls_dirs(self.jobs_dir):
            yield os.path.basename(os.path.dirname(job_path))

    def _get_builds(self, job, build_limit=sys.maxsize):
        '''Returns whether builds are precise (guarantees existence)'''
        if self.metadata.get('sequential', True):
            try:
                latest_build = int(self.get(f'{self.jobs_dir}{job}/latest-build.txt'))
            except (ValueError, TypeError):
                pass
            else:
                return False, (str(n) for n in range(latest_build, 0, -1)[:build_limit])
        # Invalid latest-build or bucket is using timestamps
        build_paths = self.ls_dirs(f'{self.jobs_dir}{job}/')
        return True, sorted(
            (os.path.basename(os.path.dirname(b)) for b in build_paths),
            key=pad_numbers, reverse=True)[:build_limit]

    def get_started_finished(self, job, build):
        if self.metadata.get('pr'):
            build_dir = self.get(f'{self.jobs_dir}/directory/{job}/{build}.txt').strip()
        else:
            build_dir = f'{self.jobs_dir}{job}/{build}'
        started = self.get(f'{build_dir}/started.json', as_json=True)
        finished = self.get(f'{build_dir}/finished.json', as_json=True)
        return build_dir, started, finished

    def get_builds(self, builds_have, build_limit=sys.maxsize):
        """Generates all (job, build) pairs ever."""
        if self.metadata.get('pr'):
            files = self.ls(self.jobs_dir + '/directory/', delim=False, build_limit=build_limit)
            for fname in files:
                if fname.endswith('.txt') and 'latest-build' not in fname:
                    job, build = fname[:-4].split('/')[-2:]
                    if (job, build) in builds_have:
                        continue
                    yield job, build
            return
        for job in self._get_jobs():
            if job in self.metadata.get('exclude_jobs', []):
                continue
            have = 0
            precise, builds = self._get_builds(job, build_limit)
            for build in builds:
                if (job, build) in builds_have:
                    have += 1
                    if have > 40 and not precise:
                        break
                    continue
                yield job, build

def mp_init_worker(jobs_dir, metadata, client_class, use_signal=True):
    """
    Initialize the environment for multiprocessing-based multithreading.
    """

    if use_signal:
        signal.signal(signal.SIGINT, signal.SIG_IGN)
    # Multiprocessing doesn't allow local variables for each worker, so we need
    # to make a GCSClient global variable.
    global WORKER_CLIENT  # pylint: disable=global-statement
    WORKER_CLIENT = client_class(jobs_dir, metadata)

def get_started_finished(job_info):
    (job, build) = job_info
    try:
        return WORKER_CLIENT.get_started_finished(job, build)
    except: # pylint: disable=W0702
        logging.exception('failed to get tests for %s/%s', job, build)
        return None, None, None

def get_junits(build_info):
    (build_id, gcs_path) = build_info
    try:
        junits = WORKER_CLIENT.get_junits_from_build(gcs_path)
        return build_id, gcs_path, junits
    except:
        logging.exception('failed to get junits for %s', gcs_path)
        raise

def get_all_builds(db, jobs_dir, metadata, threads, client_class, build_limit):
    """
    Adds information about tests to a dictionary.

    Args:
        jobs_dir: the GCS path containing jobs.
        metadata: a dict of metadata about the jobs_dir.
        threads: how many threads to use to download build information.
        client_class: a constructor for a GCSClient (or a subclass).
    """
    gcs = client_class(jobs_dir, metadata)

    print(f'Loading builds from {jobs_dir}')
    sys.stdout.flush()

    builds_have = db.get_existing_builds(jobs_dir)
    print(f'already have {len(builds_have)} builds')
    sys.stdout.flush()

    jobs_and_builds = gcs.get_builds(builds_have, build_limit)
    pool = None
    if threads > 1:
        pool = multiprocessing.Pool(threads, mp_init_worker,
                                    (jobs_dir, metadata, client_class))
        builds_iterator = pool.imap_unordered(
            get_started_finished, jobs_and_builds)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = gcs
        builds_iterator = (
            get_started_finished(job_build) for job_build in jobs_and_builds)

    try:
        for n, (build_dir, started, finished) in enumerate(builds_iterator):
            if not build_dir:
                continue # skip builds that raised exceptions
            print(f'inserting build: {build_dir}')
            if started or finished:
                db.insert_build(build_dir, started, finished)
            if n % 200 == 0:
                db.commit()
    except KeyboardInterrupt:
        if pool:
            pool.terminate()
        raise
    else:
        if pool:
            pool.close()
            pool.join()
    db.commit()

def remove_system_out(data):
    """Strip bloated system-out annotations."""
    if 'system-out' in data:
        try:
            root = ET.fromstring(data)
            for parent in root.findall('*//system-out/..'):
                for child in parent.findall('system-out'):
                    parent.remove(child)
            return ET.tostring(root, 'unicode')
        except ET.ParseError:
            pass
    return data

def download_junit(db, threads, client_class):
    """Download junit results for builds without them."""
    logging.info('Downloading JUnit artifacts.')
    sys.stdout.flush()
    builds_to_grab = db.get_builds_missing_junit()
    pool = None
    if threads > 1:
        pool = multiprocessing.pool.ThreadPool(
            threads, mp_init_worker, ('', {}, client_class, False))
        test_iterator = pool.imap_unordered(
            get_junits, builds_to_grab)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = client_class('', {})
        test_iterator = (
            get_junits(build_path) for build_path in builds_to_grab)
    for n, (build_id, build_path, junits) in enumerate(test_iterator, 1):
        logging.info('%d/%d %s %d %d', n, len(builds_to_grab),
                     build_path, len(junits), len(''.join(junits.values())))
        junits = {k: remove_system_out(v) for k, v in junits.items()}

        db.insert_build_junits(build_id, junits)
        if n % 100 == 0:
            db.commit()
    db.commit()
    if pool:
        pool.close()
        pool.join()

def main(db, jobs_dirs, threads, get_junit, build_limit, client_class=GCSClient):
    """Collect test info in matching jobs."""
    get_all_builds(db, 'gs://kubernetes-jenkins/pr-logs', {'pr': True},
                   threads, client_class, build_limit)
    for bucket, metadata in jobs_dirs.items():
        if not bucket.endswith('/'):
            bucket += '/'
        get_all_builds(db, bucket, metadata, threads, client_class, build_limit)
    if get_junit:
        download_junit(db, threads, client_class)

def get_options(argv):
    """Process command line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--buckets',
        help='YAML file with GCS bucket locations',
        required=True,
    )
    parser.add_argument(
        '--threads',
        help='number of concurrent threads to download results with',
        default=32,
        type=int,
    )
    parser.add_argument(
        '--junit',
        action='store_true',
        help='Download JUnit results from each build'
    )
    parser.add_argument(
        '--buildlimit',
        help='maximum number of runs within each job to pull, \
         all jobs will be collected if unset or 0',
        default=int(os.getenv('BUILD_LIMIT', '0')),
        type=int,
    )
    return parser.parse_args(argv)

if __name__ == '__main__':
    OPTIONS = get_options(sys.argv[1:])
    OPTIONS.buildlimit = OPTIONS.buildlimit or sys.maxsize
    main(
        model.Database(),
        yaml.safe_load(open(OPTIONS.buckets)),
        OPTIONS.threads,
        OPTIONS.junit,
        OPTIONS.buildlimit,
        )
<commit_msg>kettle: setup logging for make_db<commit_after># Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generates a SQLite DB containing test data downloaded from GCS."""

import argparse
import logging
import json
import os
import random
import re
import signal
import sys
import time
import urllib.parse
from xml.etree import cElementTree as ET

import multiprocessing
import multiprocessing.pool
import requests
import ruamel.yaml as yaml

import model

def pad_numbers(string):
    """Modify a string to make its numbers suitable for natural sorting."""
    return re.sub(r'\d+', lambda m: m.group(0).rjust(16, '0'), string)

WORKER_CLIENT = None  # used for multiprocessing

class GCSClient:
    def __init__(self, jobs_dir, metadata=None):
        self.jobs_dir = jobs_dir
        self.metadata = metadata or {}
        self.session = requests.Session()

    def _request(self, path, params, as_json=True):
        """GETs a JSON resource from GCS, with retries on failure.

        Retries are based on guidance from
        cloud.google.com/storage/docs/gsutil/addlhelp/RetryHandlingStrategy

        """
        url = f'https://www.googleapis.com/storage/v1/b/{path}'
        for retry in range(23):
            try:
                resp = self.session.get(url, params=params, stream=False)
                if 400 <= resp.status_code < 500 and resp.status_code != 429:
                    return None
                resp.raise_for_status()
                if as_json:
                    try:
                        return resp.json()
                    except json.decoder.JSONDecodeError:
                        logging.error('Failed to decode request for %s',
                                      urllib.parse.unquote(url))
                        return None
                return resp.text
            except requests.exceptions.RequestException:
                logging.exception('request failed %s', url)
            time.sleep(random.random() * min(60, 2 ** retry))

    @staticmethod
    def _parse_uri(path):
        if not path.startswith('gs://'):
            raise ValueError("Bad path: %s, should start with 'gs://'" % path)
        bucket, prefix = path[5:].split('/', 1)
        return bucket, prefix

    def get(self, path, as_json=False):
        """Get an object from GCS."""
        bucket, prefix = self._parse_uri(path)
        return self._request(f'{bucket}/o/{urllib.parse.quote(prefix, "")}',
                             {'alt': 'media'}, as_json=as_json)

    def ls(self,
           path,
           dirs=True,
           files=True,
           delim=True,
           item_field='name',
           build_limit=sys.maxsize,):
        """Lists objects under a path on gcs."""
        # pylint: disable=invalid-name

        bucket, prefix = self._parse_uri(path)
        params = {'prefix': prefix, 'fields': 'nextPageToken'}
        if delim:
            params['delimiter'] = '/'
            if dirs:
                params['fields'] += ',prefixes'
        if files:
            params['fields'] += f',items({item_field})'
        while build_limit > 0:
            resp = self._request(f'{bucket}/o', params)
            if resp is None:  # nothing under path?
                return
            for prefix in resp.get('prefixes', []):
                build_limit -= 1
                yield f'gs://{bucket}/{prefix}'
            for item in resp.get('items', []):
                if item_field == 'name':
                    build_limit -= 1
                    yield f'gs://{bucket}/{item["name"]}'
                else:
                    build_limit -= 1
                    yield item[item_field]
            if 'nextPageToken' not in resp:
                break
            params['pageToken'] = resp['nextPageToken']

    def ls_dirs(self, path):
        return self.ls(path, dirs=True, files=False)

    def _ls_junit_paths(self, build_dir):
        """Lists the paths of JUnit XML files for a build."""
        url = f'{build_dir}artifacts/'
        for path in self.ls(url):
            if re.match(r'.*/junit.*\.xml$', path):
                yield path

    def get_junits_from_build(self, build_dir):
        """Generates all tests for a build."""
        files = {}
        assert not build_dir.endswith('/')
        for junit_path in self._ls_junit_paths(build_dir + '/'):
            junit = self.get(junit_path)
            if junit is None:
                continue
            files[junit_path] = junit
        return files

    def _get_jobs(self):
        """Generates all jobs in the bucket."""
        for job_path in self.ls_dirs(self.jobs_dir):
            yield os.path.basename(os.path.dirname(job_path))

    def _get_builds(self, job, build_limit=sys.maxsize):
        '''Returns whether builds are precise (guarantees existence)'''
        if self.metadata.get('sequential', True):
            try:
                latest_build = int(self.get(f'{self.jobs_dir}{job}/latest-build.txt'))
            except (ValueError, TypeError):
                pass
            else:
                return False, (str(n) for n in range(latest_build, 0, -1)[:build_limit])
        # Invalid latest-build or bucket is using timestamps
        build_paths = self.ls_dirs(f'{self.jobs_dir}{job}/')
        return True, sorted(
            (os.path.basename(os.path.dirname(b)) for b in build_paths),
            key=pad_numbers, reverse=True)[:build_limit]

    def get_started_finished(self, job, build):
        if self.metadata.get('pr'):
            build_dir = self.get(f'{self.jobs_dir}/directory/{job}/{build}.txt').strip()
        else:
            build_dir = f'{self.jobs_dir}{job}/{build}'
        started = self.get(f'{build_dir}/started.json', as_json=True)
        finished = self.get(f'{build_dir}/finished.json', as_json=True)
        return build_dir, started, finished

    def get_builds(self, builds_have, build_limit=sys.maxsize):
        """Generates all (job, build) pairs ever."""
        if self.metadata.get('pr'):
            files = self.ls(self.jobs_dir + '/directory/', delim=False, build_limit=build_limit)
            for fname in files:
                if fname.endswith('.txt') and 'latest-build' not in fname:
                    job, build = fname[:-4].split('/')[-2:]
                    if (job, build) in builds_have:
                        continue
                    yield job, build
            return
        for job in self._get_jobs():
            if job in self.metadata.get('exclude_jobs', []):
                continue
            have = 0
            precise, builds = self._get_builds(job, build_limit)
            for build in builds:
                if (job, build) in builds_have:
                    have += 1
                    if have > 40 and not precise:
                        break
                    continue
                yield job, build

def mp_init_worker(jobs_dir, metadata, client_class, use_signal=True):
    """
    Initialize the environment for multiprocessing-based multithreading.
    """

    if use_signal:
        signal.signal(signal.SIGINT, signal.SIG_IGN)
    # Multiprocessing doesn't allow local variables for each worker, so we need
    # to make a GCSClient global variable.
    global WORKER_CLIENT  # pylint: disable=global-statement
    WORKER_CLIENT = client_class(jobs_dir, metadata)

def get_started_finished(job_info):
    (job, build) = job_info
    try:
        return WORKER_CLIENT.get_started_finished(job, build)
    except: # pylint: disable=W0702
        logging.exception('failed to get tests for %s/%s', job, build)
        return None, None, None

def get_junits(build_info):
    (build_id, gcs_path) = build_info
    try:
        junits = WORKER_CLIENT.get_junits_from_build(gcs_path)
        return build_id, gcs_path, junits
    except:
        logging.exception('failed to get junits for %s', gcs_path)
        raise

def get_all_builds(db, jobs_dir, metadata, threads, client_class, build_limit):
    """
    Adds information about tests to a dictionary.

    Args:
        jobs_dir: the GCS path containing jobs.
        metadata: a dict of metadata about the jobs_dir.
        threads: how many threads to use to download build information.
        client_class: a constructor for a GCSClient (or a subclass).
    """
    gcs = client_class(jobs_dir, metadata)

    logging.info(f'Loading builds from {jobs_dir}')
    sys.stdout.flush()

    builds_have = db.get_existing_builds(jobs_dir)
    logging.info(f'already have {len(builds_have)} builds')
    sys.stdout.flush()

    jobs_and_builds = gcs.get_builds(builds_have, build_limit)
    pool = None
    if threads > 1:
        pool = multiprocessing.Pool(threads, mp_init_worker,
                                    (jobs_dir, metadata, client_class))
        builds_iterator = pool.imap_unordered(
            get_started_finished, jobs_and_builds)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = gcs
        builds_iterator = (
            get_started_finished(job_build) for job_build in jobs_and_builds)

    try:
        for n, (build_dir, started, finished) in enumerate(builds_iterator):
            if not build_dir:
                continue # skip builds that raised exceptions
            logging.info(f'inserting build: {build_dir}')
            if started or finished:
                db.insert_build(build_dir, started, finished)
            if n % 200 == 0:
                db.commit()
    except KeyboardInterrupt:
        if pool:
            pool.terminate()
        raise
    else:
        if pool:
            pool.close()
            pool.join()
    db.commit()

def remove_system_out(data):
    """Strip bloated system-out annotations."""
    if 'system-out' in data:
        try:
            root = ET.fromstring(data)
            for parent in root.findall('*//system-out/..'):
                for child in parent.findall('system-out'):
                    parent.remove(child)
            return ET.tostring(root, 'unicode')
        except ET.ParseError:
            pass
    return data

def download_junit(db, threads, client_class):
    """Download junit results for builds without them."""
    logging.info('Downloading JUnit artifacts.')
    sys.stdout.flush()
    builds_to_grab = db.get_builds_missing_junit()
    pool = None
    if threads > 1:
        pool = multiprocessing.pool.ThreadPool(
            threads, mp_init_worker, ('', {}, client_class, False))
        test_iterator = pool.imap_unordered(
            get_junits, builds_to_grab)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = client_class('', {})
        test_iterator = (
            get_junits(build_path) for build_path in builds_to_grab)
    for n, (build_id, build_path, junits) in enumerate(test_iterator, 1):
        logging.info('%d/%d %s %d %d', n, len(builds_to_grab),
                     build_path, len(junits), len(''.join(junits.values())))
        junits = {k: remove_system_out(v) for k, v in junits.items()}

        db.insert_build_junits(build_id, junits)
        if n % 100 == 0:
            db.commit()
    db.commit()
    if pool:
        pool.close()
        pool.join()

def main(db, jobs_dirs, threads, get_junit, build_limit, client_class=GCSClient):
    """Collect test info in matching jobs."""
    setup_logging()
    get_all_builds(db, 'gs://kubernetes-jenkins/pr-logs', {'pr': True},
                   threads, client_class, build_limit)
    for bucket, metadata in jobs_dirs.items():
        if not bucket.endswith('/'):
            bucket += '/'
        get_all_builds(db, bucket, metadata, threads, client_class, build_limit)
    if get_junit:
        download_junit(db, threads, client_class)

def setup_logging():
    """Initialize logging to screen"""
    # See https://docs.python.org/2/library/logging.html#logrecord-attributes
    # [IWEF]mmdd HH:MM:SS.mmm] msg
    fmt = '%(levelname).1s%(asctime)s.%(msecs)03d] %(message)s'  # pylint: disable=line-too-long
    datefmt = '%m%d %H:%M:%S'
    logging.basicConfig(
        level=logging.INFO,
        format=fmt,
        datefmt=datefmt,
    )

def get_options(argv):
    """Process command line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--buckets',
        help='YAML file with GCS bucket locations',
        required=True,
    )
    parser.add_argument(
        '--threads',
        help='number of concurrent threads to download results with',
        default=32,
        type=int,
    )
    parser.add_argument(
        '--junit',
        action='store_true',
        help='Download JUnit results from each build'
    )
    parser.add_argument(
        '--buildlimit',
        help='maximum number of runs within each job to pull, \
         all jobs will be collected if unset or 0',
        default=int(os.getenv('BUILD_LIMIT', '0')),
        type=int,
    )
    return parser.parse_args(argv)

if __name__ == '__main__':
    OPTIONS = get_options(sys.argv[1:])
    OPTIONS.buildlimit = OPTIONS.buildlimit or sys.maxsize
    main(
        model.Database(),
        yaml.safe_load(open(OPTIONS.buckets)),
        OPTIONS.threads,
        OPTIONS.junit,
        OPTIONS.buildlimit,
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Keyboard layout generator.

Usage:
    layout.py <rows> <columns> <keycodefile> [--filter=<tag>] <layoutfile>
    layout.py (-h | --help)
    layout.py --version

Options:
    -h --help       Show this screen.
    --version       Show version.
    --filter=<tag>  Include only layout sections matching <tag>

"""

#   Copyright 2016 Daniel P. Seemuth
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import docopt
import sys

__version__ = '0.1.0'

__title__ = 'keyboardlayout'
__author__ = 'Daniel P. Seemuth'
__description__ = 'Keyboard layout generator for keyboards from uniquekeyboard.com'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Daniel P. Seemuth'

class KeyType:
    NORMAL = 0
    FN = 1
    SPACEFN = 2
    SWITCHLAYER = 3
    MEDIA = 4
    TOGGLEFN = 5

class Key:
    """One key's assignment: value and type.

    """

    def __init__(self, keyval, keytype=KeyType.NORMAL):
        self.keyval = keyval
        self.keytype = keytype

    def __repr__(self):
        return 'Key({:}, {:})'.format(self.keyval, self.keytype)

class Layer:
    """Hold all key assignments for one layer.

    """

    def __init__(self, size, keys=None):
        self.rows, self.columns = size
        self.keys = [
            [None for c in range(self.columns)]
            for r in range(self.rows)
        ]

        if keys:
            for coords, key in keys:
                x, y = coords
                self.keys[y][x] = key

    def commands(self, z):
        """Yield one uniqueksetkey command per key in the layout.

        """

        for r in range(self.rows):
            for c in range(self.columns):
                key = self.keys[r][c]

                if key is None:
                    key = Key(0, KeyType.NORMAL)

                yield 'uniqueksetkey({:}({:}({:}({:}({:}'.format(
                    c, r, z, key.keyval, key.keytype,
                )

def keycodesfromfile(file):
    ret = dict()

    for line in file:
        line = line.rstrip()

        if not line:
            continue

        try:
            name, keyval, keytype = line.split('\t')
        except ValueError:
            print('Invalid line:', line)
            continue

        keyval = int(keyval)
        keytype = int(keytype)

        ret[name] = Key(keyval, keytype)

    return ret

def layoutfromfile(file, layersize, filter, keycodes):
    unknowncodes = set()
    layers = dict()
    curlayer = None

    for linenum, line in enumerate(file, start=1):
        line = line.rstrip()

        if not line:
            continue

        if line.upper().startswith('LAYOUT '):
            parts = line.split()
            if len(parts) < 3:
                raise ValueError(
                    'invalid layer format',
                    linenum,
                    line,
                )

            layernum = int(parts[1])
            tag = parts[2]

            if (filter is None) or (tag == filter):
                if layernum in layers:
                    raise ValueError(
                        'layer already defined',
                        linenum,
                        layernum,
                    )

                curlayer = Layer(layersize)
                layers[layernum] = curlayer
                row = 0

            else:
                curlayer = None

            continue

        if curlayer is None:
            continue

        if row >= layersize[0]:
            raise ValueError(
                'too many rows',
                linenum,
                row,
            )

        parts = line.split('\t')

        for col, keyname in enumerate(parts):
            if not keyname:
                continue

            key = None
            if keyname in keycodes:
                key = keycodes[keyname]

            elif keyname.startswith('CUSTOM_'):
                subparts = keyname.split('_')
                if len(subparts) != 3:
                    raise ValueError(
                        'invalid CUSTOM key format',
                        linenum,
                        keyname,
                    )

                key = Key(
                    int(subparts[1]),
                    int(subparts[2]),
                )

            if key is not None:
                curlayer.keys[row][col] = key
            else:
                unknowncodes.add(keyname)

        row += 1

    return layers, unknowncodes

if __name__ == '__main__':
    args = docopt.docopt(__doc__, version=__version__)

    rows = int(args['<rows>'])
    columns = int(args['<columns>'])

    with open(args['<keycodefile>'], 'r') as F:
        keycodes = keycodesfromfile(F)

    with open(args['<layoutfile>'], 'r') as F:
        layers, unknowncodes = layoutfromfile(
            F,
            (rows, columns),
            args['--filter'],
            keycodes,
        )

    for code in sorted(unknowncodes):
        print('UNKNOWN', repr(code))
<commit_msg>Print keyboard layout setting commands for all layers<commit_after>#!/usr/bin/env python3

"""
Keyboard layout generator.

Usage:
    layout.py <rows> <columns> <keycodefile> [--filter=<tag>] <layoutfile>
    layout.py (-h | --help)
    layout.py --version

Options:
    -h --help       Show this screen.
    --version       Show version.
    --filter=<tag>  Include only layout sections matching <tag>

"""

#   Copyright 2016 Daniel P. Seemuth
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import docopt
import sys

__version__ = '0.1.0'

__title__ = 'keyboardlayout'
__author__ = 'Daniel P. Seemuth'
__description__ = 'Keyboard layout generator for keyboards from uniquekeyboard.com'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Daniel P. Seemuth'

class KeyType:
    NORMAL = 0
    FN = 1
    SPACEFN = 2
    SWITCHLAYER = 3
    MEDIA = 4
    TOGGLEFN = 5

class Key:
    """One key's assignment: value and type.

    """

    def __init__(self, keyval, keytype=KeyType.NORMAL):
        self.keyval = keyval
        self.keytype = keytype

    def __repr__(self):
        return 'Key({:}, {:})'.format(self.keyval, self.keytype)

class Layer:
    """Hold all key assignments for one layer.

    """

    def __init__(self, size, keys=None):
        self.rows, self.columns = size
        self.keys = [
            [None for c in range(self.columns)]
            for r in range(self.rows)
        ]

        if keys:
            for coords, key in keys:
                x, y = coords
                self.keys[y][x] = key

    def commands(self, z):
        """Yield one uniqueksetkey command per key in the layout.

        """

        for r in range(self.rows):
            for c in range(self.columns):
                key = self.keys[r][c]

                if key is None:
                    key = Key(0, KeyType.NORMAL)

                yield 'uniqueksetkey({:}({:}({:}({:}({:}'.format(
                    c, r, z, key.keyval, key.keytype,
                )

def keycodesfromfile(file):
    ret = dict()

    for line in file:
        line = line.rstrip()

        if not line:
            continue

        try:
            name, keyval, keytype = line.split('\t')
        except ValueError:
            print('Invalid line:', line)
            continue

        keyval = int(keyval)
        keytype = int(keytype)

        ret[name] = Key(keyval, keytype)

    return ret

def layoutfromfile(file, layersize, filter, keycodes):
    unknowncodes = set()
    layers = dict()
    curlayer = None

    for linenum, line in enumerate(file, start=1):
        line = line.rstrip()

        if not line:
            continue

        if line.upper().startswith('LAYOUT '):
            parts = line.split()
            if len(parts) < 3:
                raise ValueError(
                    'invalid layer format',
                    linenum,
                    line,
                )

            layernum = int(parts[1])
            tag = parts[2]

            if (filter is None) or (tag == filter):
                if layernum in layers:
                    raise ValueError(
                        'layer already defined',
                        linenum,
                        layernum,
                    )

                curlayer = Layer(layersize)
                layers[layernum] = curlayer
                row = 0

            else:
                curlayer = None

            continue

        if curlayer is None:
            continue

        if row >= layersize[0]:
            raise ValueError(
                'too many rows',
                linenum,
                row,
            )

        parts = line.split('\t')

        for col, keyname in enumerate(parts):
            if not keyname:
                continue

            key = None
            if keyname in keycodes:
                key = keycodes[keyname]

            elif keyname.startswith('CUSTOM_'):
                subparts = keyname.split('_')
                if len(subparts) != 3:
                    raise ValueError(
                        'invalid CUSTOM key format',
                        linenum,
                        keyname,
                    )

                key = Key(
                    int(subparts[1]),
                    int(subparts[2]),
                )

            if key is not None:
                curlayer.keys[row][col] = key
            else:
                unknowncodes.add(keyname)

        row += 1

    return layers, unknowncodes

if __name__ == '__main__':
    args = docopt.docopt(__doc__, version=__version__)

    rows = int(args['<rows>'])
    columns = int(args['<columns>'])

    with open(args['<keycodefile>'], 'r') as F:
        keycodes = keycodesfromfile(F)

    with open(args['<layoutfile>'], 'r') as F:
        layers, unknowncodes = layoutfromfile(
            F,
            (rows, columns),
            args['--filter'],
            keycodes,
        )

    for layernum, layer in sorted(layers.items()):
        for command in layer.commands(layernum):
            print(command, end=' ')
        print()

    for code in sorted(unknowncodes):
        print('UNKNOWN', repr(code))
<|endoftext|>


<commit_before>import logging
import sys

from cliff.app import App

import cliff.commandmanager

import instances
import logs
import properties

class CliApp(App):

    log = logging.getLogger(__name__)

    def __init__(self):
      command_manager = cliff.commandmanager.CommandManager('jxaas')
      command_manager.add_command('list-properties', properties.ListProperties)
      command_manager.add_command('list-log', logs.ListLog)
      command_manager.add_command('list-instances', instances.ListInstances)
      command_manager.add_command('destroy-instance', instances.DestroyInstance)
      command_manager.add_command('create-instance', instances.CreateInstance)
      command_manager.add_command('connect-instance', instances.ConnectInstance)
      # command_manager.add_command('complete', cliff.complete.CompleteCommand)
      super(CliApp, self).__init__(
            description='JXaaS CLI app',
            version='0.1',
            command_manager=command_manager
            )

    def initialize_app(self, argv):
        self.log.debug('initialize_app')

    def prepare_to_run_command(self, cmd):
        self.log.debug('prepare_to_run_command %s', cmd.__class__.__name__)

    def clean_up(self, cmd, result, err):
        self.log.debug('clean_up %s', cmd.__class__.__name__)
        if err:
            self.log.debug('got an error: %s', err)

def main(argv=sys.argv[1:]):
    app = CliApp()
    return app.run(argv)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<commit_msg>Turn down requests/urllib3 logging<commit_after>import logging
import sys

from cliff.app import App

import cliff.commandmanager

import instances
import logs
import properties

class CliApp(App):

    log = logging.getLogger(__name__)

    def __init__(self):
      command_manager = cliff.commandmanager.CommandManager('jxaas')
      command_manager.add_command('list-properties', properties.ListProperties)
      command_manager.add_command('list-log', logs.ListLog)
      command_manager.add_command('list-instances', instances.ListInstances)
      command_manager.add_command('destroy-instance', instances.DestroyInstance)
      command_manager.add_command('create-instance', instances.CreateInstance)
      command_manager.add_command('connect-instance', instances.ConnectInstance)
      # command_manager.add_command('complete', cliff.complete.CompleteCommand)
      super(CliApp, self).__init__(
            description='JXaaS CLI app',
            version='0.1',
            command_manager=command_manager
            )

    def initialize_app(self, argv):
        self.log.debug('initialize_app')

    def prepare_to_run_command(self, cmd):
        self.log.debug('prepare_to_run_command %s', cmd.__class__.__name__)

    def clean_up(self, cmd, result, err):
        self.log.debug('clean_up %s', cmd.__class__.__name__)
        if err:
            self.log.debug('got an error: %s', err)

    def configure_logging(self):
        super(CliApp, self).configure_logging()

        logging.getLogger("requests").setLevel(logging.WARNING)
        logging.getLogger("urllib3").setLevel(logging.WARNING)

def main(argv=sys.argv[1:]):
    app = CliApp()
    return app.run(argv)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8
from __future__ import absolute_import

import logging
from pprint import pformat

import requests
from requests import RequestException

from . import config

LOG = logging.getLogger(__name__)
LOG.addHandler(logging.NullHandler())

class K8sClientException(RequestException):
    pass

class NotFound(K8sClientException):
    """The resource was not found, and the operation could not be completed"""

class ServerError(K8sClientException):
    """The API-server returned an internal error"""

class ClientError(K8sClientException):
    """The client made a bad request"""

class Client(object):
    _session = requests.Session()

    @classmethod
    def init_session(cls):
        if "Authorization" not in cls._session.headers and config.api_token:
            cls._session.headers.update({"Authorization": "Bearer {}".format(config.api_token)})
        if cls._session.cert is None and config.cert:
            cls._session.cert = config.cert
        cls._session.verify = config.verify_ssl

    def get(self, url, **kwargs):
        return self._call("GET", url, **kwargs)

    def delete(self, url):
        return self._call("DELETE", url)

    def post(self, url, body):
        return self._call("POST", url, body)

    def put(self, url, body):
        return self._call("PUT", url, body)

    def _call(self, method, url, body=None, **kwargs):
        self.init_session()
        resp = self._session.request(method, config.api_server + url, json=body, timeout=10, **kwargs)
        self._raise_on_status(resp)
        return resp

    @staticmethod
    def _raise_on_status(resp):
        if resp.status_code == 404:
            raise NotFound(response=resp)
        elif 400 <= resp.status_code < 500:
            http_error_msg = '{0:d} Client Error: {1:s} for url: {2:s}'.format(
                    resp.status_code, resp.reason, resp.url)
            try:
                json_response = resp.json()
                LOG.debug("Client error request: %s", Client._format_request(resp.request))
                LOG.debug("Client error response: %s", pformat(json_response))
                causes = json_response.get(u"details", {}).get(u"causes", {})
                lines = ["{}: {}".format(d[u"field"], d[u"message"]) for d in causes]
                http_error_msg += '\nCauses: \n\t{0:s}'.format("\n\t".join(lines))
            except Exception as e:
                LOG.debug("Exception when dealing with client error response: %s", e)
            raise ClientError(http_error_msg, response=resp)

        elif 500 <= resp.status_code < 600:
            http_error_msg = '%s Server Error: %s for url: %s' % (resp.status_code, resp.reason, resp.url)
            raise ServerError(http_error_msg, response=resp)

    @staticmethod
    def _format_request(request):
        return "Request({})".format(pformat({
            "method": request.method,
            "url": request.url,
            "headers": request.headers,
            "body": request.body
        }))
<commit_msg>INFRA-1009 - Fix e2e test for ingress<commit_after>#!/usr/bin/env python
# -*- coding: utf-8
from __future__ import absolute_import

import logging
from pprint import pformat

import requests
from requests import RequestException

from . import config

LOG = logging.getLogger(__name__)
LOG.addHandler(logging.NullHandler())

class K8sClientException(RequestException):
    pass

class NotFound(K8sClientException):
    """The resource was not found, and the operation could not be completed"""

class ServerError(K8sClientException):
    """The API-server returned an internal error"""

class ClientError(K8sClientException):
    """The client made a bad request"""

class Client(object):
    _session = requests.Session()

    @classmethod
    def init_session(cls):
        if "Authorization" not in cls._session.headers and config.api_token:
            cls._session.headers.update({"Authorization": "Bearer {}".format(config.api_token)})
        if cls._session.cert is None and config.cert:
            cls._session.cert = config.cert
        cls._session.verify = config.verify_ssl

    def get(self, url, **kwargs):
        return self._call("GET", url, **kwargs)

    def delete(self, url):
        return self._call("DELETE", url)

    def post(self, url, body):
        return self._call("POST", url, body)

    def put(self, url, body):
        return self._call("PUT", url, body)

    def _call(self, method, url, body=None, **kwargs):
        self.init_session()
        resp = self._session.request(method, config.api_server + url, json=body, timeout=10, **kwargs)
        self._raise_on_status(resp)
        return resp

    @staticmethod
    def _raise_on_status(resp):
        if resp.status_code < 400:
            return
        elif resp.status_code == 404:
            exc = NotFound
        elif 400 <= resp.status_code < 500:
            exc = ClientError
        else:
            exc = ServerError
        http_error_msg = '{:d} {:s}: {:s} for url: {:s}'.format(
                resp.status_code, exc.__name__, resp.reason, resp.url)
        LOG.debug("Request: %s", Client._format_request(resp.request))
        try:
            json_response = resp.json()
            LOG.debug("Response: %s", pformat(json_response))
            causes = json_response.get(u"details", {}).get(u"causes", {})
            lines = ["{}: {}".format(d[u"field"], d[u"message"]) for d in causes]
            http_error_msg += '\nCauses: \n\t{0:s}'.format("\n\t".join(lines))
        except Exception as e:
            LOG.debug("Exception when dealing with client error response: %s", e)
            LOG.debug("Response: %r", resp.text)
        raise exc(http_error_msg, response=resp)

    @staticmethod
    def _format_request(request):
        return "Request({})".format(pformat({
            "method": request.method,
            "url": request.url,
            "headers": request.headers,
            "body": request.body
        }))
<|endoftext|>


<commit_before># Filename: logger.py
# pylint: disable=locally-disabled,C0103
"""
The logging facility.

"""
from hashlib import sha256
import socket
import logging

from .tools import colored, supports_color

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

loggers = {}  # this holds all the registered loggers
# logging.basicConfig()

if supports_color():
    logging.addLevelName(logging.INFO, "\033[1;32m%s\033[1;0m" %
                         logging.getLevelName(logging.INFO))
    logging.addLevelName(logging.DEBUG, "\033[1;34m%s\033[1;0m" %
                         logging.getLevelName(logging.DEBUG))
    logging.addLevelName(logging.WARNING, "\033[1;33m%s\033[1;0m" %
                         logging.getLevelName(logging.WARNING))
    logging.addLevelName(logging.ERROR, "\033[1;31m%s\033[1;0m" %
                         logging.getLevelName(logging.ERROR))
    logging.addLevelName(logging.CRITICAL, "\033[1;101m%s\033[1;0m" %
                         logging.getLevelName(logging.CRITICAL))

class LogIO(object):
    """Read/write logging information.
    """

    def __init__(self, node, stream,
                 url='pi2089.physik.uni-erlangen.de',
                 port=28777):
        self.node = node
        self.stream = stream
        self.url = url
        self.port = port
        self.sock = None
        self.connect()

    def send(self, message, level='info'):
        message_string = "+log|{0}|{1}|{2}|{3}\r\n" \
                         .format(self.stream, self.node, level, message)
        try:
            self.sock.send(message_string)
        except socket.error:
            print("Lost connection, reconnecting...")
            self.connect()
            self.sock.send(message_string)

    def connect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.url, self.port))

def get_logger(name):
    """Helper function to get a logger"""
    if name in loggers:
        return loggers[name]
    logger = logging.getLogger(name)
    logger.propagate = False
    pre1, suf1 = hash_coloured_escapes(name) if supports_color() else ('', '')
    pre2, suf2 = hash_coloured_escapes(name + 'salt')  \
                 if supports_color() else ('', '')
    formatter = logging.Formatter('%(levelname)s {}●{}●{} %(name)s: %(message)s'
                                  .format(pre1, pre2, suf1))
    ch = logging.StreamHandler()
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    loggers[name] = logger
    return logger

def set_level(name, level):
    """Set the log level for given logger"""
    get_logger(name).setLevel(level)

def get_printer(name, color=None, ansi_code=None, force_color=False):
    """Return a function which prints a message with a coloured name prefix"""

    if force_color or supports_color():
        if color is None and ansi_code is None:
            cpre_1, csuf_1 = hash_coloured_escapes(name)
            cpre_2, csuf_2 = hash_coloured_escapes(name + 'salt')
            name = cpre_1 + '●' + cpre_2 + '●' + csuf_1 + ' ' + name
        else:
            name = colored(name, color=color, ansi_code=ansi_code)

    prefix = name + ': '

    def printer(text):
        print(prefix + str(text))

    return printer

def hash_coloured(text):
    """Return a ANSI coloured text based on its hash"""
    ansi_code = int(sha256(text.encode('utf-8')).hexdigest(), 16) % 230
    return colored(text, ansi_code=ansi_code)

def hash_coloured_escapes(text):
    """Return the ANSI hash colour prefix and suffix for a given text"""
    ansi_code = int(sha256(text.encode('utf-8')).hexdigest(), 16) % 230
    prefix, suffix = colored('SPLIT', ansi_code=ansi_code).split('SPLIT')
    return prefix, suffix
<commit_msg>Revert "Improve logger/printer colours"<commit_after># Filename: logger.py
# pylint: disable=locally-disabled,C0103
"""
The logging facility.

"""
from hashlib import sha256
import socket
import logging

from .tools import colored, supports_color

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

loggers = {}  # this holds all the registered loggers
# logging.basicConfig()

if supports_color():
    logging.addLevelName(logging.INFO, "\033[1;32m%s\033[1;0m" %
                         logging.getLevelName(logging.INFO))
    logging.addLevelName(logging.DEBUG, "\033[1;34m%s\033[1;0m" %
                         logging.getLevelName(logging.DEBUG))
    logging.addLevelName(logging.WARNING, "\033[1;33m%s\033[1;0m" %
                         logging.getLevelName(logging.WARNING))
    logging.addLevelName(logging.ERROR, "\033[1;31m%s\033[1;0m" %
                         logging.getLevelName(logging.ERROR))
    logging.addLevelName(logging.CRITICAL, "\033[1;101m%s\033[1;0m" %
                         logging.getLevelName(logging.CRITICAL))

class LogIO(object):
    """Read/write logging information.
    """

    def __init__(self, node, stream,
                 url='pi2089.physik.uni-erlangen.de',
                 port=28777):
        self.node = node
        self.stream = stream
        self.url = url
        self.port = port
        self.sock = None
        self.connect()

    def send(self, message, level='info'):
        message_string = "+log|{0}|{1}|{2}|{3}\r\n" \
                         .format(self.stream, self.node, level, message)
        try:
            self.sock.send(message_string)
        except socket.error:
            print("Lost connection, reconnecting...")
            self.connect()
            self.sock.send(message_string)

    def connect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.url, self.port))

def get_logger(name):
    """Helper function to get a logger"""
    if name in loggers:
        return loggers[name]
    logger = logging.getLogger(name)
    logger.propagate = False
    pre, suf = hash_coloured_escapes(name) if supports_color() else ('', '')
    formatter = logging.Formatter('%(levelname)s->{}%(name)s:{} %(message)s'
                                  .format(pre, suf))
    ch = logging.StreamHandler()
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    loggers[name] = logger
    return logger

def set_level(name, level):
    """Set the log level for given logger"""
    get_logger(name).setLevel(level)

def get_printer(name, color=None, ansi_code=None, force_color=False):
    """Return a function which prints a message with a coloured name prefix"""

    if force_color or supports_color():
        if color is None and ansi_code is None:
            name = hash_coloured(name)
        else:
            name = colored(name, color=color, ansi_code=ansi_code)

    prefix = name + ': '

    def printer(text):
        print(prefix + str(text))

    return printer

def hash_coloured(text):
    """Return a ANSI coloured text based on its hash"""
    ansi_code = int(sha256(text.encode('utf-8')).hexdigest(), 16) % 230
    return colored(text, ansi_code=ansi_code)

def hash_coloured_escapes(text):
    """Return the ANSI hash colour prefix and suffix for a given text"""
    ansi_code = int(sha256(text.encode('utf-8')).hexdigest(), 16) % 230
    prefix, suffix = colored('SPLIT', ansi_code=ansi_code).split('SPLIT')
    return prefix, suffix
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from sys import argv, exit
import tempfile
import os.path
import subprocess

tpl = """
.equ ENC_PAYLOAD_ADDR, {payload_addr}
.equ ENC_PAYLOAD_SIZE, {payload_size}
.equ BASE, {sysmem_base}
.equ SECOND_PAYLOAD, {second_payload}
"""

prefix = "arm-vita-eabi-"

def build(code):
    with tempfile.TemporaryDirectory() as tmp:
        src_file = os.path.join(tmp, "rop.S")
        obj_file = os.path.join(tmp, "rop.o")
        bin_file = os.path.join(tmp, "rop.bin")
        fout = open(src_file, "wb")
        fout.write(code)
        fout.close()

        subprocess.check_call([prefix + "as", src_file, "-o", obj_file])
        subprocess.check_call([prefix + "objcopy", "-O", "binary", obj_file, bin_file])

        fin = open(bin_file, "rb")
        data = fin.read()
        fin.close()
        return data

def chunk(b, size):
    if len(b) % size != 0:
        raise RuntimeError("chunk: b % size != 0")
    return [b[x * size:(x + 1) * size] for x in range(0, len(b) // size)]

def write_c_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos):
    output = "unsigned krop[] = {"
    output += ", ".join(hex(int.from_bytes(x, 'little')) for x in krop)
    output += "};\nunsigned relocs[] = {"
    output += ", ".join(str(x) for x in relocs)
    output += "};\n"
    output += "krop[{}] = ENC_PAYLOAD_ADDR;\n".format(addr_pos)
    output += "krop[{}] = (ENC_PAYLOAD_SIZE >> 2) + 0x10;\n".format(size_shift_pos)
    output += "krop[{}] = ENC_PAYLOAD_SIZE ^ 0x40;\n".format(size_xor_pos)
    output += "krop[{}] = ENC_PAYLOAD_SIZE;\n".format(size_plain_pos)
    output += "krop[{}] = SECOND_PAYLOAD;\n".format(second_payload_pos)
    return output

def write_rop_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos):
    output = ""
    tpl = "store({}, krop + 0x{:x});\n"
    for x, (addr, reloc) in enumerate(zip(krop, relocs)):
        addr = int.from_bytes(addr, "little")
        if reloc == 0:
            s = "0x{:x}".format(addr)
        else:
            output += "add([sysmem_base], 0x{:x});\n".format(addr)
            s = "&return"
        output += tpl.format(s, x * 4)
    output += "store([kx_loader_addr], krop + 0x{:x});\n".format(addr_pos * 4)
    # I've hardcoded payload size to be 0x200, deal with it
    payload_size = 0x200
    output += "store(0x{:x}, krop + 0x{:x});\n".format((payload_size >> 2) + 0x10, size_shift_pos * 4)
    output += "store(0x{:x}, krop + 0x{:x});\n".format(payload_size ^ 0x40, size_xor_pos * 4)
    output += "store(0x{:x}, krop + 0x{:x});\n".format(payload_size, size_plain_pos * 4)
    output += "store(second_payload, krop + 0x{:x});\n".format(second_payload_pos * 4)
    return output

def main():
    if len(argv) != 3:
        print("Usage: build_rop.py rop.S output-directory/")
        return -1
    fin = open(argv[1], "rb")
    code = fin.read()
    fin.close()

    tags = {
        "payload_addr": 0xF0F0F0F0,
        "payload_size": 0x0A0A0A00,
        "sysmem_base": 0xB0B00000,
        "second_payload": 0xC0C0C0C0,
    }

    first = build(tpl.format(payload_addr=0, payload_size=0, sysmem_base=0, second_payload=0).encode("ascii") + code)
    second = build(tpl.format(**tags).encode("ascii") + code)

    if len(first) != len(second):
        print("wtf? got different krop lengths")
        return -2

    # Find differences in krops, a difference indicates either that this address depends on sysmem base or it's
    # payload addr/size
    krop = first = chunk(first, 4)
    second = chunk(second, 4)
    relocs = [0] * len(first)
    addr_pos = size_shift_pos = size_xor_pos = size_plain_pos = second_payload_pos = -1
    for i, (first_word, second_word) in enumerate(zip(first, second)):
        if first_word != second_word:
            second = int.from_bytes(second_word, "little")
            if second == tags["payload_addr"]:
                addr_pos = i
            elif second == (tags["payload_size"] >> 2) + 0x10:
                size_shift_pos = i
            elif second == tags["payload_size"] ^ 0x40:
                size_xor_pos = i
            elif second == tags["payload_size"]:
                size_plain_pos = i
            elif second == tags["second_payload"]:
                second_payload_pos = i
            else:
                relocs[i] = 1

    if -1 in [addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos]:
        print("unable to resolve positions: addr={}, size_shift={}, size_xor={}, size_plain={}, second_payload={}".format(
            addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))
        return -2

    print("Kernel rop size: 0x{:x} bytes".format(len(krop) * 4))

    with open(os.path.join(argv[2], "krop.c"), "w") as fout:
        fout.write(write_c_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))
    with open(os.path.join(argv[2], "krop.rop"), "w") as fout:
        fout.write(write_rop_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))

if __name__ == "__main__":
    exit(main())
<commit_msg>build: add ropfuscator step<commit_after>#!/usr/bin/env python3
from sys import argv, exit
import tempfile
import os.path
import subprocess

tpl = """
.equ ENC_PAYLOAD_ADDR, {payload_addr}
.equ ENC_PAYLOAD_SIZE, {payload_size}
.equ BASE, {sysmem_base}
.equ SECOND_PAYLOAD, {second_payload}
"""

prefix = "arm-vita-eabi-"

def build(code):
    with tempfile.TemporaryDirectory() as tmp:
        src_file = os.path.join(tmp, "rop.S")
        obj_file = os.path.join(tmp, "rop.o")
        bin_file = os.path.join(tmp, "rop.bin")
        obf_file = os.path.join(tmp, "rop.obf")
        fout = open(src_file, "wb")
        fout.write(code)
        fout.close()

        subprocess.check_call([prefix + "as", src_file, "-o", obj_file])
        subprocess.check_call([prefix + "objcopy", "-O", "binary", obj_file, bin_file])
        subprocess.check_call(["python3", "krop/ropfuscator.py", bin_file, obf_file, "DxT9HVn5"])

        fin = open(obf_file, "rb")
        data = fin.read()
        fin.close()
        return data

def chunk(b, size):
    if len(b) % size != 0:
        raise RuntimeError("chunk: b % size != 0")
    return [b[x * size:(x + 1) * size] for x in range(0, len(b) // size)]

def write_c_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos):
    output = "unsigned krop[] = {"
    output += ", ".join(hex(int.from_bytes(x, 'little')) for x in krop)
    output += "};\nunsigned relocs[] = {"
    output += ", ".join(str(x) for x in relocs)
    output += "};\n"
    output += "krop[{}] = ENC_PAYLOAD_ADDR;\n".format(addr_pos)
    output += "krop[{}] = (ENC_PAYLOAD_SIZE >> 2) + 0x10;\n".format(size_shift_pos)
    output += "krop[{}] = ENC_PAYLOAD_SIZE ^ 0x40;\n".format(size_xor_pos)
    output += "krop[{}] = ENC_PAYLOAD_SIZE;\n".format(size_plain_pos)
    output += "krop[{}] = SECOND_PAYLOAD;\n".format(second_payload_pos)
    return output

def write_rop_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos):
    output = ""
    tpl = "store({}, krop + 0x{:x});\n"
    for x, (addr, reloc) in enumerate(zip(krop, relocs)):
        addr = int.from_bytes(addr, "little")
        if reloc == 0:
            s = "0x{:x}".format(addr)
        else:
            output += "add([sysmem_base], 0x{:x});\n".format(addr)
            s = "&return"
        output += tpl.format(s, x * 4)
    output += "store([kx_loader_addr], krop + 0x{:x});\n".format(addr_pos * 4)
    # I've hardcoded payload size to be 0x200, deal with it
    payload_size = 0x200
    output += "store(0x{:x}, krop + 0x{:x});\n".format((payload_size >> 2) + 0x10, size_shift_pos * 4)
    output += "store(0x{:x}, krop + 0x{:x});\n".format(payload_size ^ 0x40, size_xor_pos * 4)
    output += "store(0x{:x}, krop + 0x{:x});\n".format(payload_size, size_plain_pos * 4)
    output += "store(second_payload, krop + 0x{:x});\n".format(second_payload_pos * 4)
    return output

def main():
    if len(argv) != 3:
        print("Usage: build_rop.py rop.S output-directory/")
        return -1
    fin = open(argv[1], "rb")
    code = fin.read()
    fin.close()

    tags = {
        "payload_addr": 0xF0F0F0F0,
        "payload_size": 0x0A0A0A00,
        "sysmem_base": 0xB0B00000,
        "second_payload": 0xC0C0C0C0,
    }

    first = build(tpl.format(payload_addr=0, payload_size=0, sysmem_base=0, second_payload=0).encode("ascii") + code)
    second = build(tpl.format(**tags).encode("ascii") + code)

    if len(first) != len(second):
        print("wtf? got different krop lengths")
        return -2

    # Find differences in krops, a difference indicates either that this address depends on sysmem base or it's
    # payload addr/size
    krop = first = chunk(first, 4)
    second = chunk(second, 4)
    relocs = [0] * len(first)
    addr_pos = size_shift_pos = size_xor_pos = size_plain_pos = second_payload_pos = -1
    for i, (first_word, second_word) in enumerate(zip(first, second)):
        if first_word != second_word:
            second = int.from_bytes(second_word, "little")
            if second == tags["payload_addr"]:
                addr_pos = i
            elif second == (tags["payload_size"] >> 2) + 0x10:
                size_shift_pos = i
            elif second == tags["payload_size"] ^ 0x40:
                size_xor_pos = i
            elif second == tags["payload_size"]:
                size_plain_pos = i
            elif second == tags["second_payload"]:
                second_payload_pos = i
            else:
                relocs[i] = 1

    if -1 in [addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos]:
        print("unable to resolve positions: addr={}, size_shift={}, size_xor={}, size_plain={}, second_payload={}".format(
            addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))
        return -2

    print("Kernel rop size: 0x{:x} bytes".format(len(krop) * 4))

    with open(os.path.join(argv[2], "krop.c"), "w") as fout:
        fout.write(write_c_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))
    with open(os.path.join(argv[2], "krop.rop"), "w") as fout:
        fout.write(write_rop_code(krop, relocs, addr_pos, size_shift_pos, size_xor_pos, size_plain_pos, second_payload_pos))

if __name__ == "__main__":
    exit(main())
<|endoftext|>


<commit_before>#
# Copyright (C) 2021  Dr. Maha Farhat
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
A second map with Marginal Resistance data.
"""

import re
import json
from collections import defaultdict
from django.views.generic import TemplateView
from django.db.models import Count

from apps.mutations.models import Drug

from .mixins import JsonView, DataSlicerMixin
from .utils import GraphData, adjust_coords
from .models import Country

from apps.maptables.models import CustomMap, MapRow

class AntibiogramMap(TemplateView):
    title = "Antibiograms Map"
    template_name = 'maps/map_antibiograms.html'

class MarginalPlaces(JsonView, DataSlicerMixin):
    """
    Provide a json output that slices mutation data by country
    """
    model = MapRow
    order = ['country__name', 'country__region']
    values = ['country__iso2', 'drug__code', 'drug__name', 'data']
    filters = {
        'parent_map__slug': ['antibiograms'],
        'drug[]': 'drug__code',
    }

    def get_context_data(self, **_):
        drugs = dict()
        rows = defaultdict(dict)
        for row in self.get_data():
            drugs[row['drug__code']] = row['drug__name']
            rows[row['country__iso2']][row['drug__code']] = json.loads(row['data'])

        # Limit to one drug only
        drug = list(drugs)[0]

        return {
            "type": "FeatureCollection",
            'filters': self.applied_filters(),
            'drugs': drugs,
            'features': [
                {
                    # Turning this to json and then back to python just to feed
                    # to JsonView, seems a little wasteful and redundent.
                    "geometry": adjust_coords(json.loads(country.geom.geojson)),
                    "popupContent": country.name,
                    "type": "Feature",
                    "id": country.id,
                    "properties": {
                        "name": country.name,
                        "value": country.iso2,
                        "row": rows[country.iso2][drug],
                        "drug": {'name': drugs[drug], 'code': drug},
                    }

                } for country in Country.objects.filter(iso2__in=rows)
            ]
        }

class MarginalDrugs(JsonView, DataSlicerMixin):
    """Provide a json data slice into the drug resistance data"""
    model = MapRow
    order = ['drug__regimen', '-drug__priority',]
    values = ['drug__name', 'drug__code', 'data']
    filters = {
        'parent_map__slug': ['antibiograms'],
        'country[]': 'country__iso2__in',
    }

    def get_context_data(self, **_):
        """Return a dictionary of template variables"""

        totals = defaultdict(int)
        for row in self.get_data():
            dat = json.loads(row['data'])
            totals[row['drug__code']] += dat['gentb_snp10_n']

        drug_dict = [
            {
                'key': 'gentb_snp10_n',
                'values': [{
                    'y': i,
                    'x': t,
                    'col': t,
                    'value': i,
                    'total': i,
                } for t, i in totals.items()]
            }
        ]

        return {
            'data': drug_dict,
            'filters': self.applied_filters(),
        }

<commit_msg>Adjust coords back for live<commit_after>#
# Copyright (C) 2021  Dr. Maha Farhat
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
A second map with Marginal Resistance data.
"""

import re
import json
from collections import defaultdict
from django.views.generic import TemplateView
from django.db.models import Count

from apps.mutations.models import Drug

from .mixins import JsonView, DataSlicerMixin
from .utils import GraphData, adjust_coords
from .models import Country

from apps.maptables.models import CustomMap, MapRow

class AntibiogramMap(TemplateView):
    title = "Antibiograms Map"
    template_name = 'maps/map_antibiograms.html'

class MarginalPlaces(JsonView, DataSlicerMixin):
    """
    Provide a json output that slices mutation data by country
    """
    model = MapRow
    order = ['country__name', 'country__region']
    values = ['country__iso2', 'drug__code', 'drug__name', 'data']
    filters = {
        'parent_map__slug': ['antibiograms'],
        'drug[]': 'drug__code',
    }

    def get_context_data(self, **_):
        drugs = dict()
        rows = defaultdict(dict)
        for row in self.get_data():
            drugs[row['drug__code']] = row['drug__name']
            rows[row['country__iso2']][row['drug__code']] = json.loads(row['data'])

        # Limit to one drug only
        drug = list(drugs)[0]

        return {
            "type": "FeatureCollection",
            'filters': self.applied_filters(),
            'drugs': drugs,
            'features': [
                {
                    # Turning this to json and then back to python just to feed
                    # to JsonView, seems a little wasteful and redundent.
                    #"geometry": adjust_coords(json.loads(country.geom.geojson)),
                    "geometry": json.loads(country.geom.geojson),
                    "popupContent": country.name,
                    "type": "Feature",
                    "id": country.id,
                    "properties": {
                        "name": country.name,
                        "value": country.iso2,
                        "row": rows[country.iso2][drug],
                        "drug": {'name': drugs[drug], 'code': drug},
                    }

                } for country in Country.objects.filter(iso2__in=rows)
            ]
        }

class MarginalDrugs(JsonView, DataSlicerMixin):
    """Provide a json data slice into the drug resistance data"""
    model = MapRow
    order = ['drug__regimen', '-drug__priority',]
    values = ['drug__name', 'drug__code', 'data']
    filters = {
        'parent_map__slug': ['antibiograms'],
        'country[]': 'country__iso2__in',
    }

    def get_context_data(self, **_):
        """Return a dictionary of template variables"""

        totals = defaultdict(int)
        for row in self.get_data():
            dat = json.loads(row['data'])
            totals[row['drug__code']] += dat['gentb_snp10_n']

        drug_dict = [
            {
                'key': 'gentb_snp10_n',
                'values': [{
                    'y': i,
                    'x': t,
                    'col': t,
                    'value': i,
                    'total': i,
                } for t, i in totals.items()]
            }
        ]

        return {
            'data': drug_dict,
            'filters': self.applied_filters(),
        }

<|endoftext|>


<commit_before>import logging
import optparse

from .channel import LazyChannel
from .queue import Queue
from .config import Config

logger = logging.getLogger(__name__)

def main():
    logging.basicConfig(level=logging.INFO)

    parser = optparse.OptionParser()
    parser.add_option('--config')
    options, args = parser.parse_args()

    config = Config(options.config)
    channel = LazyChannel(
        config.RABBIT_HOST, config.RABBIT_PORT,
        config.RABBIT_USER, config.RABBIT_PASSWORD)
    channel.tx_select()
    failed_queue = Queue('kuyruk_failed', channel)

    while 1:
        message = failed_queue.receive()
        if message is None:
            break

        tag, task_description = message
        queue_name = task_description['queue']
        del task_description['queue']
        task_queue = Queue(queue_name, channel)
        task_queue.send(task_description)

        failed_queue.ack(tag)
        channel.tx_commit()

    logger.info('All failed task are requeued.')

if __name__ == '__main__':
    main()
<commit_msg>fix requeue<commit_after>import logging
import optparse

from .channel import LazyChannel
from .queue import Queue
from .config import Config

logger = logging.getLogger(__name__)

def main():
    logging.basicConfig(level=logging.INFO)

    parser = optparse.OptionParser()
    parser.add_option('--config')
    options, args = parser.parse_args()

    config = Config(options.config)
    channel = LazyChannel(
        config.RABBIT_HOST, config.RABBIT_PORT,
        config.RABBIT_USER, config.RABBIT_PASSWORD)
    channel.tx_select()
    failed_queue = Queue('kuyruk_failed', channel)

    while 1:
        tag, task_description = failed_queue.receive()
        if task_description is None:
            break

        queue_name = task_description['queue']
        del task_description['queue']
        task_queue = Queue(queue_name, channel)
        task_queue.send(task_description)

        failed_queue.ack(tag)
        channel.tx_commit()

    logger.info('All failed task are requeued.')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before><commit_msg>Dont bother saving bunk error<commit_after><|endoftext|>


<commit_before># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from setuptools import find_packages
from setuptools import setup

PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))

with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj:
    README = file_obj.read()

# NOTE: This is duplicated throughout and we should try to
#       consolidate.
SETUP_BASE = {
    'author': 'Google Cloud Platform',
    'author_email': 'jjg+google-cloud-python@google.com',
    'scripts': [],
    'url': 'https://github.com/GoogleCloudPlatform/google-cloud-python',
    'license': 'Apache 2.0',
    'platforms': 'Posix; MacOS X; Windows',
    'include_package_data': True,
    'zip_safe': False,
    'classifiers': [
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Topic :: Internet',
    ],
}

REQUIREMENTS = [
    'google-cloud-core >= 0.24.0, < 0.25dev',
]

setup(
    name='google-cloud-language',
    version='0.24.1',
    description='Python Client for Google Cloud Natural Language',
    long_description=README,
    namespace_packages=[
        'google',
        'google.cloud',
    ],
    packages=find_packages(exclude=('tests*',)),
    install_requires=REQUIREMENTS,
    **SETUP_BASE
)
<commit_msg>Prep language-0.25.0 release. (#3535)<commit_after># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from setuptools import find_packages
from setuptools import setup

PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))

with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj:
    README = file_obj.read()

# NOTE: This is duplicated throughout and we should try to
#       consolidate.
SETUP_BASE = {
    'author': 'Google Cloud Platform',
    'author_email': 'jjg+google-cloud-python@google.com',
    'scripts': [],
    'url': 'https://github.com/GoogleCloudPlatform/google-cloud-python',
    'license': 'Apache 2.0',
    'platforms': 'Posix; MacOS X; Windows',
    'include_package_data': True,
    'zip_safe': False,
    'classifiers': [
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Topic :: Internet',
    ],
}

REQUIREMENTS = [
    'google-cloud-core >= 0.25.0, < 0.26dev',
]

setup(
    name='google-cloud-language',
    version='0.25.0',
    description='Python Client for Google Cloud Natural Language',
    long_description=README,
    namespace_packages=[
        'google',
        'google.cloud',
    ],
    packages=find_packages(exclude=('tests*',)),
    install_requires=REQUIREMENTS,
    **SETUP_BASE
)
<|endoftext|>