<commit_before>#!/usr/bin/python
# <variable> = required
# Usage ./lava-report.py <option> [json]
import argparse
import subprocess
from utils import *

def parse_json(json):
    jobs = load_json(json)
    url = validate_input(jobs['username'], jobs['token'], jobs['server'])
    connection = connect(url)
    duration = jobs['duration']
    # Remove unused data
    jobs.pop('duration')
    jobs.pop('username')
    jobs.pop('token')
    jobs.pop('server')
    return connection, jobs, duration

def boot_report(args):
    connection, jobs, duration =  parse_json(args.boot)
    results_directory = os.getcwd() + '/results'
    results = {}
    mkdir(results_directory)
    for job_id in jobs:
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        # Retrieve the log file
        binary_job_file = connection.scheduler.job_output(job_id)
        job_file = str(binary_job_file)
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            bundle_attributes =  bundle_data['test_runs'][0]['attributes']
            boot_meta = {}
            kernel_defconfig = None
            kernel_version = None
            kernel_endian = None
            kernel_boot_time = None
            kernel_tree = None
            kernel_image = None
            kernel_addr = None
            initrd_addr = None
            dtb_addr = None
            dtb_append = None
            fastboot = None
            fastboot_cmd = None
            if in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
            if in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                kernel_boot_time = bundle_attributes['kernel-boot-time']
            if in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if in_bundle_attributes(bundle_attributes, 'kernel-image'):
                kernel_image = bundle_attributes['kernel-image']
            if in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            print 'Creating boot log for %s' % device_type
            log = 'boot-%s.log' % device_type
            directory = os.path.join(results_directory, kernel_defconfig)
            ensure_dir(directory)
            write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            boot_meta['boot_log'] = log
            # TODO: Fix this
            boot_meta['boot_log_html'] = None
            boot_meta['boot_result'] = result
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            # TODO: Fix this
            boot_meta['dtb'] = None
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            boot_meta['fastboot_cmds'] = fastboot_cmd
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            boot_meta['kernel_image'] = kernel_image
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % device_type
            write_json(json_file, directory, boot_meta)

    if results and kernel_tree and kernel_version:
        print 'Creating boot summary for %s' % kernel_version
        log = '%s-boot-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, log), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Status Dashboard: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total duration: %.2f seconds\n' % duration)
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git describe: %s\n' % kernel_version)
            f.write('\n')
            f.write('Full Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))
        # sendmail
        cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, log)
        subprocess.check_output(cmd, shell=True)

def main(args):
    if args.boot:
        boot_report(args)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--boot", help="creates a kernel-ci boot report from a given json file")
    parser.add_argument("--email", help="email address to send report to")
    args = parser.parse_args()
    main(args)
<commit_msg>kernel-ci: cleanup e-mail reporting<commit_after>#!/usr/bin/python
# <variable> = required
# Usage ./lava-report.py <option> [json]
import argparse
import subprocess
from utils import *

def parse_json(json):
    jobs = load_json(json)
    url = validate_input(jobs['username'], jobs['token'], jobs['server'])
    connection = connect(url)
    duration = jobs['duration']
    # Remove unused data
    jobs.pop('duration')
    jobs.pop('username')
    jobs.pop('token')
    jobs.pop('server')
    return connection, jobs, duration

def boot_report(args):
    connection, jobs, duration =  parse_json(args.boot)
    results_directory = os.getcwd() + '/results'
    results = {}
    mkdir(results_directory)
    for job_id in jobs:
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        # Retrieve the log file
        binary_job_file = connection.scheduler.job_output(job_id)
        job_file = str(binary_job_file)
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            bundle_attributes =  bundle_data['test_runs'][0]['attributes']
            boot_meta = {}
            kernel_defconfig = None
            kernel_version = None
            kernel_endian = None
            kernel_boot_time = None
            kernel_tree = None
            kernel_image = None
            kernel_addr = None
            initrd_addr = None
            dtb_addr = None
            dtb_append = None
            fastboot = None
            fastboot_cmd = None
            if in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
            if in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                kernel_boot_time = bundle_attributes['kernel-boot-time']
            if in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if in_bundle_attributes(bundle_attributes, 'kernel-image'):
                kernel_image = bundle_attributes['kernel-image']
            if in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            print 'Creating boot log for %s' % device_type
            log = 'boot-%s.log' % device_type
            directory = os.path.join(results_directory, kernel_defconfig)
            ensure_dir(directory)
            write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': device_type, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            boot_meta['boot_log'] = log
            # TODO: Fix this
            boot_meta['boot_log_html'] = None
            boot_meta['boot_result'] = result
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            # TODO: Fix this
            boot_meta['dtb'] = None
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            boot_meta['fastboot_cmds'] = fastboot_cmd
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            boot_meta['kernel_image'] = kernel_image
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % device_type
            write_json(json_file, directory, boot_meta)

    if results and kernel_tree and kernel_version:
        print 'Creating boot summary for %s' % kernel_version
        log = '%s-boot-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, log), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Status Dashboard: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total duration: %.2f seconds\n' % duration)
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git describe: %s\n' % kernel_version)
            f.write('\n')
            f.write('Full Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))
        # sendmail
        print 'Sending e-mail summary to %s' % args.email
        cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, log)
        subprocess.check_output(cmd, shell=True)

def main(args):
    if args.boot:
        boot_report(args)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--boot", help="creates a kernel-ci boot report from a given json file")
    parser.add_argument("--email", help="email address to send report to")
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation
#

def pytest_addoption(parser):
    parser.addoption(
        "--delete_indexes",
        action='store_true',
        default=False,
        help="A flag to indicate if we should delete all indexes in ES before each test. This could result in inconsistent tests if not specified."
    )
    parser.addoption(
        "--delete_queues",
        action='store_true',
        default=False,
        help="A flag to indicate if we should delete the contents of rabbitmq queues. This could result in inconsistent tests if not specified."
    )

def pytest_configure(config):
    warning_text = ""
    if not config.option.delete_indexes:
        warning_text += "\n** WARNING - Some unit tests will not pass unless the --delete_indexes is specified."
        warning_text += "\nThis is due to the fact that some tests need a 'clean' ES environment **\n"
        warning_text += "** DISCLAIMER - If you enable this flag, all indexes that MozDef uses will be deleted upon test execution **\n"
    else:
        warning_text += "\n** WARNING - The --delete_indexes flag has been set. We will be deleting important indexes from ES before test execution**\n"
        warning_text += "Continuing the unit test execution in 10 seconds...CANCEL ME IF YOU DO NOT WANT PREVIOUS INDEXES DELETED!!! **\n"

    if not config.option.delete_queues:
        warning_text += "\n** WARNING - Some unit tests will not pass unless the --delete_queues is specified."
        warning_text += "\nThis is due to the fact that some tests need a 'clean' RabbitMQ environment **\n"
        warning_text += "** DISCLAIMER - If you enable this flag, the queues in rabbitmq that MozDef uses will be deleted upon test execution **\n"
    else:
        warning_text += "\n** WARNING - The --delete_queues flag has been set. We will be purging RabbitMQ queues before test execution**\n"
        warning_text += "Continuing the unit test execution in 10 seconds...CANCEL ME IF YOU DO NOT WANT PREVIOUS QUEUES PURGED!!! **\n"

    print warning_text
    time.sleep(10)
<commit_msg>Fix breaking tests<commit_after>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

import time

def pytest_addoption(parser):
    parser.addoption(
        "--delete_indexes",
        action='store_true',
        default=False,
        help="A flag to indicate if we should delete all indexes in ES before each test. This could result in inconsistent tests if not specified."
    )
    parser.addoption(
        "--delete_queues",
        action='store_true',
        default=False,
        help="A flag to indicate if we should delete the contents of rabbitmq queues. This could result in inconsistent tests if not specified."
    )

def pytest_configure(config):
    warning_text = ""
    if not config.option.delete_indexes:
        warning_text += "\n** WARNING - Some unit tests will not pass unless the --delete_indexes is specified."
        warning_text += "\nThis is due to the fact that some tests need a 'clean' ES environment **\n"
        warning_text += "** DISCLAIMER - If you enable this flag, all indexes that MozDef uses will be deleted upon test execution **\n"
    else:
        warning_text += "\n** WARNING - The --delete_indexes flag has been set. We will be deleting important indexes from ES before test execution**\n"
        warning_text += "Continuing the unit test execution in 10 seconds...CANCEL ME IF YOU DO NOT WANT PREVIOUS INDEXES DELETED!!! **\n"

    if not config.option.delete_queues:
        warning_text += "\n** WARNING - Some unit tests will not pass unless the --delete_queues is specified."
        warning_text += "\nThis is due to the fact that some tests need a 'clean' RabbitMQ environment **\n"
        warning_text += "** DISCLAIMER - If you enable this flag, the queues in rabbitmq that MozDef uses will be deleted upon test execution **\n"
    else:
        warning_text += "\n** WARNING - The --delete_queues flag has been set. We will be purging RabbitMQ queues before test execution**\n"
        warning_text += "Continuing the unit test execution in 10 seconds...CANCEL ME IF YOU DO NOT WANT PREVIOUS QUEUES PURGED!!! **\n"

    print warning_text
    time.sleep(10)
<|endoftext|>


<commit_before>import json
from io import BytesIO
from pprint import pprint  # noqa

from aleph.model import Document
from aleph.tests.util import TestCase
from aleph.queues import get_status, get_stage, OP_INGEST, OP_PROCESS
from aleph.logic.processing import process_collection

class IngestApiTestCase(TestCase):

    def setUp(self):
        super(IngestApiTestCase, self).setUp()
        self.rolex = self.create_user(foreign_id='user_3')
        self.col = self.create_collection()
        self.url = '/api/2/collections/%s/ingest' % self.col.id

    def test_upload_logged_out(self):
        data = {'meta': json.dumps({})}
        res = self.client.post(self.url,
                               data=data)
        assert res.status_code == 403, res

    def test_upload_no_meta(self):
        _, headers = self.login(is_admin=True)
        data = {'meta': 'hihi'}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_upload_csv_doc(self):
        _, headers = self.login(is_admin=True)
        meta = {
            'collection_id': self.col.id,
            'countries': ['de', 'usa'],
            'languages': ['en'],
            'mime_type': 'text/csv',
            'source_url': 'http://pudo.org/experts.csv'
        }
        csv_path = self.get_fixture_path('experts.csv')
        data = {
            'meta': json.dumps(meta),
            'foo': open(csv_path, 'rb'),
        }
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, (res, res.data)
        assert 'id' in res.json, res.json

        db_id, _ = res.json.get('id').split('.', 1)
        doc = Document.by_id(db_id)
        assert doc.schema == Document.SCHEMA, doc.schema
        assert doc.meta['countries'] == ['de', 'us'], doc.meta
        assert doc.meta['languages'] == ['eng'], doc.meta

        status = get_status(self.col)
        assert status.get('pending') == 1, status
        job = status.get('jobs')[0]
        assert job.get('pending') == 1, job
        stage = job.get('stages')[0]
        assert stage.get('stage') == OP_INGEST, stage
        assert stage.get('pending') == 1, stage

    def test_invalid_meta(self):
        _, headers = self.login(is_admin=True)
        meta = {'title': 3, 'file_name': ''}
        data = {
            'meta': json.dumps(meta),
            'foo': (BytesIO(b"this is a futz with a banana"), 'futz.html')
        }
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_invalid_directory_without_foreign_id(self):
        _, headers = self.login(is_admin=True)
        meta = {'title': 'test'}
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_directory_with_file(self):
        _, headers = self.login(is_admin=True)
        meta = {
            'file_name': 'directory',
            'foreign_id': 'directory',
            'schema': 'Folder',
            'collection_id': self.col.id,
        }
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, res
        assert 'id' in res.json, res.json
        directory = res.json['id']

        meta = {
            'file_name': 'subdirectory',
            'foreign_id': 'subdirectory',
            'parent': {'id': directory},
            'collection_id': self.col.id,
        }
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, res
        stage = get_stage(self.col, OP_PROCESS)
        process_collection(stage, self.col, ingest=False)
        assert 'id' in res.json, res.json
        url = '/api/2/entities/%s' % res.json['id']
        res = self.client.get(url, headers=headers)
        assert res.status_code == 200, res
        props = res.json.get('properties')
        assert 'subdirectory' in props['fileName'], res.json
<commit_msg>Remove unnecessary collection id metadata<commit_after>import json
from io import BytesIO
from pprint import pprint  # noqa

from aleph.model import Document
from aleph.tests.util import TestCase
from aleph.queues import get_status, get_stage, OP_INGEST, OP_PROCESS
from aleph.logic.processing import process_collection

class IngestApiTestCase(TestCase):

    def setUp(self):
        super(IngestApiTestCase, self).setUp()
        self.rolex = self.create_user(foreign_id='user_3')
        self.col = self.create_collection()
        self.url = '/api/2/collections/%s/ingest' % self.col.id

    def test_upload_logged_out(self):
        data = {'meta': json.dumps({})}
        res = self.client.post(self.url,
                               data=data)
        assert res.status_code == 403, res

    def test_upload_no_meta(self):
        _, headers = self.login(is_admin=True)
        data = {'meta': 'hihi'}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_upload_csv_doc(self):
        _, headers = self.login(is_admin=True)
        meta = {
            'countries': ['de', 'usa'],
            'languages': ['en'],
            'mime_type': 'text/csv',
            'source_url': 'http://pudo.org/experts.csv'
        }
        csv_path = self.get_fixture_path('experts.csv')
        data = {
            'meta': json.dumps(meta),
            'foo': open(csv_path, 'rb'),
        }
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, (res, res.data)
        assert 'id' in res.json, res.json

        db_id, _ = res.json.get('id').split('.', 1)
        doc = Document.by_id(db_id)
        assert doc.schema == Document.SCHEMA, doc.schema
        assert doc.meta['countries'] == ['de', 'us'], doc.meta
        assert doc.meta['languages'] == ['eng'], doc.meta

        status = get_status(self.col)
        assert status.get('pending') == 1, status
        job = status.get('jobs')[0]
        assert job.get('pending') == 1, job
        stage = job.get('stages')[0]
        assert stage.get('stage') == OP_INGEST, stage
        assert stage.get('pending') == 1, stage

    def test_invalid_meta(self):
        _, headers = self.login(is_admin=True)
        meta = {'title': 3, 'file_name': ''}
        data = {
            'meta': json.dumps(meta),
            'foo': (BytesIO(b"this is a futz with a banana"), 'futz.html')
        }
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_invalid_directory_without_foreign_id(self):
        _, headers = self.login(is_admin=True)
        meta = {'title': 'test'}
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 400, res

    def test_directory_with_file(self):
        _, headers = self.login(is_admin=True)
        meta = {
            'file_name': 'directory',
            'foreign_id': 'directory',
            'schema': 'Folder',
            'collection_id': self.col.id,
        }
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, res
        assert 'id' in res.json, res.json
        directory = res.json['id']

        meta = {
            'file_name': 'subdirectory',
            'foreign_id': 'subdirectory',
            'parent': {'id': directory},
            'collection_id': self.col.id,
        }
        data = {'meta': json.dumps(meta)}
        res = self.client.post(self.url, data=data, headers=headers)
        assert res.status_code == 201, res
        stage = get_stage(self.col, OP_PROCESS)
        process_collection(stage, self.col, ingest=False)
        assert 'id' in res.json, res.json
        url = '/api/2/entities/%s' % res.json['id']
        res = self.client.get(url, headers=headers)
        assert res.status_code == 200, res
        props = res.json.get('properties')
        assert 'subdirectory' in props['fileName'], res.json
<|endoftext|>


<commit_before>from aiohttp import errors, web
import asyncio
from asynctest import TestCase, Mock, patch, ignore_loop
from json import loads
from nose.tools import (
    assert_is, assert_is_not_none, assert_raises, assert_true, eq_
)

from nyuki.api import Api, mw_capability, mw_json, APIRequest
from nyuki.capabilities import Response
from tests import future_func, make_future

class TestApi(TestCase):

    def setUp(self):
        self._api = Api(self.loop)
        self._host = 'localhost'
        self._port = 8080

    @patch('aiohttp.web.Application.make_handler')
    @patch('asyncio.unix_events._UnixSelectorEventLoop.create_server')
    async def test_001_build_server(self, create_server_mock, handler_mock):
        await self._api.build(self._host, self._port)
        eq_(handler_mock.call_count, 1)
        eq_(create_server_mock.call_count, 1)
        create_server_mock.assert_called_with(
            self._api._handler, host=self._host, port=self._port
        )

    async def test_002_destroy_server(self):
        with patch.object(self._api, '_handler') as i_handler:
            i_handler.finish_connections.return_value = make_future([])
            with patch.object(self._api, '_server') as i_server:
                i_server.wait_closed.return_value = make_future([])
                with patch.object(self._api._server, 'close') as call_close:
                    await self._api.destroy()
                    eq_(call_close.call_count, 1)
                    eq_(i_server.wait_closed.call_count, 1)

    @ignore_loop
    def test_003_instantiated_router(self):
        assert_is(self._api.router, self._api._app.router)

class TestJsonMiddleware(TestCase):

    def setUp(self):
        self._request = Mock()
        self._request.POST_METHODS = ['POST', 'PUT', 'PATCH']
        self._app = Mock()

    async def test_001_request_valid_header_content_post_method(self):
        self._request.headers = {'CONTENT-TYPE': 'application/json'}
        ret_value = {'test_value': 'kikoo_test'}
        self._request.json.return_value = make_future(ret_value)
        self._request.method = 'POST'

        @future_func
        def _next_handler(r):
            return ret_value

        mdw = await mw_json(self._app, _next_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        eq_(response, ret_value)

    async def test_002_request_handling_non_post_method(self):
        self._request.headers = {}
        ret_value = True
        self._request.method = 'GET'

        @future_func
        def _next_handler(r):
            return ret_value

        mdw = await mw_json(self._app, _next_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        eq_(response, ret_value)

    async def test_003_request_invalid_or_missing_header(self):
        headers = [{}, {'CONTENT-TYPE': 'application/octet-stream'}]
        values = ['', 'this_is_octet']
        self._request.method = 'POST'

        @future_func
        def _next_handler(r):
            return 'dummy'

        for idx, h in enumerate(headers):
            self._request.headers = h
            self._request.content = values[idx]
            mdw = await mw_json(self._app, _next_handler)
            assert_is_not_none(mdw)
            if idx > 0:
                with assert_raises(errors.HttpBadRequest):
                    await mdw(self._request)

class TestCapabilityMiddleware(TestCase):

    def setUp(self):
        self._request = Mock()
        self._request.POST_METHODS = ['POST', 'PUT', 'PATCH']
        self._app = Mock()

    async def test_001a_extract_data_from_payload_post_method(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}

        @future_func
        def json():
            return {'capability': 'string_manip'}

        self._request.json = json

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 'ok'})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 'ok')
        eq_(response.status, 200)

    async def test_001b_extract_data_from_non_post_method(self):
        self._request.method = 'GET'
        self._request.GET = {'id': 2}
        self._request.match_info = {'name': 'test'}

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 2})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 2)
        eq_(response.status, 200)

    async def test_001c_post_no_data(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}

        @future_func
        def json():
            return None

        self._request.json = json

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 'ok'})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 'ok')
        eq_(response.status, 200)

    async def test_002_no_response(self):
        self._request.method = 'GET'
        self._request.GET = {}
        self._request.match_info = {}

        @future_func
        def _capa_handler(d):
            return Response()

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(response.body, None)
        eq_(response.status, 200)

    async def test_003_request_headers(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}
        self._request.headers = {'Content-Type': 'application/json'}

        @future_func
        def json():
            return {'capability': 'test'}

        self._request.json = json

        ar = await APIRequest.from_request(self._request)
        eq_(ar['capability'], 'test')
        eq_(ar.headers.get('Content-Type'), 'application/json')
<commit_msg>fix tests for API raw response<commit_after>from aiohttp import errors, web
import asyncio
from asynctest import TestCase, Mock, patch, ignore_loop
from json import loads
from nose.tools import (
    assert_is, assert_is_not_none, assert_raises, assert_true, eq_
)

from nyuki.api import Api, mw_capability, mw_json, APIRequest
from nyuki.capabilities import Response
from tests import future_func, make_future

class TestApi(TestCase):

    def setUp(self):
        self._api = Api(self.loop)
        self._host = 'localhost'
        self._port = 8080

    @patch('aiohttp.web.Application.make_handler')
    @patch('asyncio.unix_events._UnixSelectorEventLoop.create_server')
    async def test_001_build_server(self, create_server_mock, handler_mock):
        await self._api.build(self._host, self._port)
        eq_(handler_mock.call_count, 1)
        eq_(create_server_mock.call_count, 1)
        create_server_mock.assert_called_with(
            self._api._handler, host=self._host, port=self._port
        )

    async def test_002_destroy_server(self):
        with patch.object(self._api, '_handler') as i_handler:
            i_handler.finish_connections.return_value = make_future([])
            with patch.object(self._api, '_server') as i_server:
                i_server.wait_closed.return_value = make_future([])
                with patch.object(self._api._server, 'close') as call_close:
                    await self._api.destroy()
                    eq_(call_close.call_count, 1)
                    eq_(i_server.wait_closed.call_count, 1)

    @ignore_loop
    def test_003_instantiated_router(self):
        assert_is(self._api.router, self._api._app.router)

class TestJsonMiddleware(TestCase):

    def setUp(self):
        self._request = Mock()
        self._request.POST_METHODS = ['POST', 'PUT', 'PATCH']
        self._app = Mock()

    async def test_001_request_valid_header_content_post_method(self):
        self._request.headers = {'CONTENT-TYPE': 'application/json'}
        ret_value = {'test_value': 'kikoo_test'}
        self._request.json.return_value = make_future(ret_value)
        self._request.method = 'POST'

        @future_func
        def _next_handler(r):
            return ret_value

        mdw = await mw_json(self._app, _next_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        eq_(response, ret_value)

    async def test_002_request_handling_non_post_method(self):
        self._request.headers = {}
        ret_value = True
        self._request.method = 'GET'

        @future_func
        def _next_handler(r):
            return ret_value

        mdw = await mw_json(self._app, _next_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        eq_(response, ret_value)

    async def test_003_request_invalid_or_missing_header(self):
        headers = [{}, {'CONTENT-TYPE': 'application/octet-stream'}]
        values = ['', 'this_is_octet']
        self._request.method = 'POST'

        @future_func
        def _next_handler(r):
            return 'dummy'

        for idx, h in enumerate(headers):
            self._request.headers = h
            self._request.content = values[idx]
            mdw = await mw_json(self._app, _next_handler)
            assert_is_not_none(mdw)
            if idx > 0:
                with assert_raises(errors.HttpBadRequest):
                    await mdw(self._request)

class TestCapabilityMiddleware(TestCase):

    def setUp(self):
        self._request = Mock()
        self._request.POST_METHODS = ['POST', 'PUT', 'PATCH']
        self._app = Mock()
        async def json():
            return {'capability': 'test'}
        async def text():
            return '{"capability": "test"}'
        self._request.json = json
        self._request.text = text

    async def test_001a_extract_data_from_payload_post_method(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}

        @future_func
        def json():
            return {'capability': 'string_manip'}

        self._request.json = json

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 'ok'})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 'ok')
        eq_(response.status, 200)

    async def test_001b_extract_data_from_non_post_method(self):
        self._request.method = 'GET'
        self._request.GET = {'id': 2}
        self._request.match_info = {'name': 'test'}

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 2})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 2)
        eq_(response.status, 200)

    async def test_001c_post_no_data(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}

        @future_func
        def _capa_handler(d, name):
            eq_(name, 'test')
            capa_resp = Response({'response': 'ok'})
            return capa_resp

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(loads(response.body.decode('utf-8'))["response"], 'ok')
        eq_(response.status, 200)

    async def test_002_no_response(self):
        self._request.method = 'GET'
        self._request.GET = {}
        self._request.match_info = {}

        @future_func
        def _capa_handler(d):
            return Response()

        mdw = await mw_capability(self._app, _capa_handler)
        assert_is_not_none(mdw)
        response = await mdw(self._request)
        assert_true(isinstance(response, web.Response))
        eq_(response.body, None)
        eq_(response.status, 200)

    async def test_003_request_headers(self):
        self._request.method = 'POST'
        self._request.match_info = {'name': 'test'}
        self._request.headers = {'Content-Type': 'application/json'}

        ar = await APIRequest.from_request(self._request)
        eq_(ar['capability'], 'test')
        eq_(ar.headers.get('Content-Type'), 'application/json')
<|endoftext|>


<commit_before>from collections import namedtuple

import pytest

class Object:
    pass

@pytest.fixture
def object():
    return Object()
<commit_msg>Automatically skip slow tests<commit_after>import pytest

class Object:
    pass

@pytest.fixture
def object():
    return Object()

def pytest_addoption(parser):
    parser.addoption("--runslow", action="store_true", help="run slow tests")

def pytest_runtest_setup(item):
    if "slow" in item.keywords and not item.config.getoption("--runslow"):
        pytest.skip("need --runslow option to run")
<|endoftext|>


<commit_before>import argparse
import xml.etree.ElementTree as ET
from collections import namedtuple
import requests
import json
import datetime

parser = argparse.ArgumentParser(description='Analyze perf tests results and notify TP.')
parser.add_argument('-f', '--file', type=str, nargs='?', help='XML-file with test results', default='results.xml')
parser.add_argument('--tp-url', type=str, nargs='?', help='TP url')
parser.add_argument('--tp-token', type=str, nargs='?', help='TP auth token')
parser.add_argument('--build-url', type=str, nargs='?', help='Current job url')

TestStats = namedtuple("TestStats", ['tests', 'failures', 'errors', 'time'])

def get_failed_tests(xml_file):
    tree = ET.parse(xml_file)
    root = tree.getroot()

    return TestStats(**root.attrib)

def tests_failed(test_stats):
    return test_stats.tests > 0 and test_stats.failures + test_stats.errors > 0

def notify_tp(url, token, build_url):
    with requests.Session() as session:
        def get_raw(collection, filter='', include='[Id,Name]'):
            request_url = "{url}/api/v1/{collection}?format=json&token={token}&where={filter}&include={include}".format(
                url=url, token=token, collection=collection, filter=filter, include=include)

            request = session.get(request_url)

            return request.json()["Items"]

        def post_raw(collection, data):
            request_url = "{url}/api/v1/{collection}?format=json&token={token}".format(url=url, token=token,
                                                                                collection=collection)

            request = session.post(request_url, data=json.dumps(data))

            return request.json()

        bug_name = 'Performance degradation. {date}'.format(date=datetime.datetime.now().date())
        description = '<!--markdown-->[Details]({build_url})'.format(build_url=build_url)

        existing_bugs = get_raw('Bugs', "(Name eq '{bug_name}')".format(bug_name=bug_name),
                                include='[Id,Name,EntityType[Id]]')

        if len(existing_bugs) == 0:
            project = get_raw('Projects', "(Name eq 'TP3')")[0]
            bug = post_raw('Bugs', {
                'name': bug_name,
                'description': description,
                'project': {
                    'id': project['Id']
                },
                'tags': 'maintenance'
            })
        else:
            existing_bug = existing_bugs[0]
            comment = post_raw('Comments', {
                'description': description,
                'general': {
                    'id': existing_bug['Id'],
                    'entityType': {
                        'id': existing_bug['EntityType']['Id']
                    }
                }
            })

# launcher

if __name__ == '__main__':
    args, extra = parser.parse_known_args()

    test_stats = get_failed_tests(args.file)

    if tests_failed(test_stats):
        notify_tp(args.tp_url, args.tp_token, args.build_url)
<commit_msg>Fix failed tests count calculation<commit_after>import argparse
import xml.etree.ElementTree as ET
from collections import namedtuple
import requests
import json
import datetime

parser = argparse.ArgumentParser(description='Analyze perf tests results and notify TP.')
parser.add_argument('-f', '--file', type=str, nargs='?', help='XML-file with test results', default='results.xml')
parser.add_argument('--tp-url', type=str, nargs='?', help='TP url')
parser.add_argument('--tp-token', type=str, nargs='?', help='TP auth token')
parser.add_argument('--build-url', type=str, nargs='?', help='Current job url')

TestStats = namedtuple("TestStats", ['tests', 'failures', 'errors', 'time'])

def get_failed_tests(xml_file):
    tree = ET.parse(xml_file)
    root = tree.getroot()

    return TestStats(**root.attrib)

def tests_failed(test_stats):
    return int(test_stats.tests) > 0 and int(test_stats.failures) + int(test_stats.errors) > 0

def notify_tp(url, token, build_url):
    with requests.Session() as session:
        def get_raw(collection, filter='', include='[Id,Name]'):
            request_url = "{url}/api/v1/{collection}?format=json&token={token}&where={filter}&include={include}".format(
                url=url, token=token, collection=collection, filter=filter, include=include)

            request = session.get(request_url)

            return request.json()["Items"]

        def post_raw(collection, data):
            request_url = "{url}/api/v1/{collection}?format=json&token={token}".format(url=url, token=token,
                                                                                collection=collection)

            request = session.post(request_url, data=json.dumps(data))

            return request.json()

        bug_name = 'Performance degradation. {date}'.format(date=datetime.datetime.now().date())
        description = '<!--markdown-->[Details]({build_url})'.format(build_url=build_url)

        existing_bugs = get_raw('Bugs', "(Name eq '{bug_name}')".format(bug_name=bug_name),
                                include='[Id,Name,EntityType[Id]]')

        if len(existing_bugs) == 0:
            project = get_raw('Projects', "(Name eq 'TP3')")[0]
            bug = post_raw('Bugs', {
                'name': bug_name,
                'description': description,
                'project': {
                    'id': project['Id']
                },
                'tags': 'maintenance'
            })
        else:
            existing_bug = existing_bugs[0]
            comment = post_raw('Comments', {
                'description': description,
                'general': {
                    'id': existing_bug['Id'],
                    'entityType': {
                        'id': existing_bug['EntityType']['Id']
                    }
                }
            })

# launcher

if __name__ == '__main__':
    args, extra = parser.parse_known_args()

    test_stats = get_failed_tests(args.file)

    if tests_failed(test_stats):
        notify_tp(args.tp_url, args.tp_token, args.build_url)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python3

from poisson import *

class Device(object):
	"""
		property:
		[protected]	buf: buffer size (bits)
		[protected]	status: point out the device is on upstream downstream by 'U' or 'D'
		[protected]	lambd: data rate (bps)
	"""
	# def __init__(self):
	# 	self._buf = {'U':0, 'D':0};
	# 	self._status = 'D';

	@property
	def buf(self):
		return self._buf;
	@buf.setter
	def buf(self, buf):
		# pick the right key and value
		buf = {k:v for k,v in buf.items() if k is 'U' or k is 'D'}
		buf = {k:v for k,v in buf.items() if type(v) is type(1)}
		self._buf.update(buf);

	@property
	def status(self):
		return self._status;
	@status.setter
	def status(self, status):
		if status is not 'D' and status is not 'U':
			raise Exception("status value sould be 'D' or 'U'");
		self._status = status;

	# FIXME: need to consider more situation for data format and exception handling
	@property
	def lambd(self):
		return self._lambd;
	@lambd.setter
	def lambd(self, lambd):
		self._lambd = lambd;

class UE(Device):
	"""
	@property
	[protected]	buf[2]
	[protected]	status
	[protected]	lambd
	[private]	parentDevice: identity served RN
	"""
	def __init__(self, buf={'U':0, 'D':0}, status='D', lambd=None, parentDevice = None):
		self._buf = buf;
		self._status = status;
		self._lambd = lambd;
		self.__parentDevice = parentDevice;

	# FIXME: need to consider more situation for data format and exception handling
	@property
	def parentDevice(self):
		return self.__parentDevice;
	# should parent RN changable? or init in begining?
	@parentDevice.setter
	def parentDevice(self, pD):
		self.__parentDevice = pD;

class RN(Device):
	"""
	@property
	[protected]	buf[2]
	[protected]	status
	[protected]	lambd
	[private]	l_UE: a list of served UE
	"""
	# FIXME
	def __init__(self, buf={'U':0, 'D':0}, status='D', lambd=None, l_UE=None):
		self._buf = buf;
		self._status = status;
		self._lambd = None;
		self.__l_UE = l_UE;

	@property
	def l_UE(self):
		return self.__l_UE;
	@l_UE.setter
	def l_UE(self, l_UE):
		if type(l_UE) != list or type(l_UE[0]) != UE:
			raise Exception("Wrong type: " + str(l_UE));
		self.__l_UE = l_UE;
		setattr(self, '__lambd')

	# rewrite lambd method, cause RN's lambd is equal to aggregate RUE lambd
	@property
	def lambd(self):
		return self.__lambd;
	@lambd.setter
	def lambd(self):
		if self.__l_UE is None:
			print("There's no served UEs for aggregate lambd");
		self.__lambd = sum(ue.lambd for ue in self.__l_UE);
		setattr(self, '_process')
<commit_msg>add: class eNB, Device.link, error handling, uni-test<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python3

import inspect

def raiser(err): raise err if type(err) is Exception else raiser(Exception(str(err)))

class Device(object):
	def __init__(self, buf={}, status='D', link='', lambd=0):
		"""
		property:
		[protected]	buf: buffer size (bits)
		[protected]	status: identify the device is on upstream downstream by 'U' or 'D'
		[protected]	link: identify the link is backhaul or access
		[protected]	lambd: data rate (bps)
		"""
		self._buf=buf;
		self._status=status;
		self._link=link;
		self._lambd=lambd;

	@property
	def buf(self):
		return self._buf;
	@buf.setter
	def buf(self, buf):
		"""
		buffer filter by checking type, key and value first, then update
		maintain the buffer always have only two items with key 'U' and 'D'
		and the value are always type int as well
		"""
		try:
			if type(buf) is dict:
				buf = {k:v for k,v in buf.items() if k is 'U' or k is 'D'}
				buf = {k:v for k,v in buf.items() if type(v) is int}
				self._buf.update(buf);
			else: raise Exception("Buffer should be the type of dict.");
		except Exception as e: print(e);

	@property
	def status(self):
		return self._status;
	@status.setter
	def status(self, status):
		"""
		maintain status will only in the value of 'U' or 'D'
		"""
		try:
			if type(status) is str:
				status = status.upper();
				self._status = status if status == 'U' or status == 'D' else raiser(Exception("status value sould be 'U' or 'D'"));
			else: raise Exception("status value sould be str");
		except Exception as e: print(e);

	@property
	def link(self):
		return self._link;
	@link.setter
	def link(self, link):
		try: self._link = link if link == 'backhaul' or link == 'access' else raiser(Exception("link should be 'baackhaul' or 'access'"));
	except Exception as e: print(e);

	@property
	def lambd(self):
		return self._lambd;
	@lambd.setter
	def lambd(self, lambd):
		try: self._lambd = lambd if type(lambd) is int else raiser(Exception("lambd value should be int"));
		except Exception as e: print(e)

class UE(Device):
	def __init__(self, buf={}, status='D', lambd=0, parentDevice = None):
		"""
		@property
		[protected]	buf
		[protected]	status
		[protected]	link
		[protected]	lambd
		[private]	parentDevice: identity served RN
		"""
		self._buf = buf;
		self._status = status;
		self._link = 'access';
		self._lambd = lambd;
		self.__parentDevice = parentDevice;

	@property
	def parentDevice(self):
		return self.__parentDevice;
	@parentDevice.setter
	def parentDevice(self, pD):
		"""
		parent should be the type of RN object
		check the type and make sure its instance rather than class by isinstance(obj, '__class__') build-in function
		"""
		try: self.__parentDevice = pD if isinstance(pD, RN) and not inspect.isclass(pD) else raiser(Exception("parent should be type of RN instance"));
		except Exception as e: print(e)

class RN(Device):
	def __init__(self, buf={}, status='D', RUE=[]):
		"""
		@property
		[protected]	buf
		[protected]	status
		[protected]	link
		[protected]	lambd
		[private]	RUE: a list of served UE
		"""
		self._buf = buf;
		self._status = status;
		self._link = 'backhaul';
		self._lambd = 0;
		self.__RUE = RUE;

	@property
	def RUE(self):
		return self.__RUE;
	@RUE.setter
	def RUE(self, RUE):
		"""
		check the type of RUE is a list and each element is type of UE
		set the RN lambda, which is aggregate RUE lambda

		NOTE: setting lambda in this setter rather then lambd.setter to prevent from changing value directly
		# FIXME: append() would pass any value
		"""
		try:
			if type(RUE) is list:
				for i in RUE:
					if not isinstance(i, UE) or inspect.isclass(i): raise Exception("RUE should be all UE instance object");
				self.__RUE = RUE;

				# setattr(self, '__lambd')
				self._lambd = sum(ue.lambd for ue in self.__RUE);
			else: raise Exception("RUE should be the type of list")
		except Exception as e: print(e);

class eNB(Device):
	def __init__(self, buf={}, status='D', relays=None):
		"""
		@property
		[protected]	buf
		[protected]	status
		[protected]	link
		[protected]	lambd
		[private]	relays: a list of served RN
		"""
		self._buf = buf;
		self._status = status;
		self._link = 'backhaul';
		self._lambd = 0;
		self.__relays = relay;

	@property
	def relays(self):
		return self.__relays;
	@relays.setter
	def relays(self, relays):
		"""
		similar to RN.RUE setter
		"""
		try:
			if type(relays) is list:
				for i in relays:
					if not isinstance(i, RN) or inspect.isclass(i): raise Exception("relays should be all RN instance object");
				self.__relays = relays;
				self._lambd = sum(rn.lambd for rn in self.__relays);
			else: raise Exception("relays should be the type of list")
		except Exception as e: print(e);

if __name__ == '__main__':

	"""
	test
	1. class Device, buffer setter
		* [Success]	assign buf with different type compared to dict
		* [Success]	assign buf in dict type but with unnecessary key except 'U' and 'D'
		* [Success]	assign buf in dict type but with incorrect type of value compared to int
	2. class Device, status setter
		* [Success]	assign status with others type except str
		* [Success]	assign status with 'u' or 'd'
		* [Success]	assign status with others letter except 'U' or 'D'
	3. class Device, lambd setter
		* [Success]	assign lambd with non-integer value
	4. class UE, parentDevice setter
		* [Success]	assign parentDevice with the non-RN object
		* [Success]	assign parentDevice with RN class rather than instance
	5. class RN, RUE setter
		* [Success]	assign RUE with non-list object
		* [Success]	assign RUE with a list of non-UE object
		* [Success]	assign RUE with UE class object
		* [Failed]	assign by .append()
	"""

	print("Test\t\tprop/func\t\t\tresult")
	test_D = Device();
	test_UE = UE();
	test_RN = RN()
	print("====\t\t========\t\t\t========")
	print("Device\t\ttype(buf) is int\t\t", end='');
	test_D.buf = 100;
	print("Device\t\tbuf={'U':50, 'D':100, 'Q':3}\t", end='');
	test_D.buf = {'U':50, 'D':100, 'Q':3}
	print(test_D.buf)
	print("Device\t\tbuf={'U':20, 'D':100, 'Q':3}\t", end='');
	test_D.buf = {'U':20, 'D':100, 'Q':3}
	print(test_D.buf)
	print("Device\t\tstatus=None\t\t\t", end='');
	test_D.status = None;
	print("Device\t\tstatus='d'\t\t\t", end='');
	test_D.status = 'd';
	print(test_D.status)
	print("Device\t\tstatus='XD'\t\t\t", end='');
	test_D.status = 'XD';
	print("Device\t\tlambd='100'\t\t\t", end="");
	test_D.lambd='100';
	print("----\t\t--------\t\t\t--------")
	print("UE\t\tparentDevice=None\t\t", end="");
	test_UE.parentDevice = None;
	print("UE\t\tparentDevice=RN()\t\t", end="");
	test_UE.parentDevice = RN;
	print("----\t\t--------\t\t\t--------")
	print("RN\t\tRUE={}\t\t\t\t", end="")
	test_RN.RUE = {};
	print("RN\t\tRUE=[1,2,3]\t\t\t", end="")
	test_RN.RUE = [1,2,3];
	print("RN\t\tRUE=[UE1, UE]\t\t\t", end="")
	test_RN.RUE = [test_UE, UE];
	print("RN\t\tRUE=[UE1], RUE.append(UE)\t", end="");	# FIXME
	test_RN.RUE = [test_UE];
	test_RN.RUE.append(UE);
	checkType = True;
	for i in test_RN.RUE:
		if inspect.isclass(i): checkType=False;break;
	print(test_RN.RUE) if checkType else print(checkType);
<|endoftext|>


<commit_before>from aiohttp import web
import amen.audio
import pydub
import time
import asyncio
import logging
import subprocess
from . import database

# To determine the "effective length" of the last beat, we
# average the last N beats prior to it. Higher numbers give
# smoother results but may have issues with a close-out rall.
LAST_BEAT_AVG = 10

app = web.Application()

songs = []
position = 0
track_list = []

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

# ------ Helper functions for infinitely_glitch() -------

# The rate-limiting sleep will wait until the clock catches up to this point.
# We start it "ten seconds ago" so we get a bit of buffer to start off.
rendered_until = time.time() - 10
ffmpeg = None # aio subprocess where we're compressing to MP3
async def _render_output_audio(seg, fn):
	logging.info("Sending %d bytes of data for %s secs of %s", len(seg.raw_data), seg.duration_seconds, fn)
	ffmpeg.stdin.write(seg.raw_data)
	await ffmpeg.stdin.drain()
	global rendered_until; rendered_until += seg.duration_seconds
	delay = rendered_until - time.time()
	if delay > 0:
		logging.debug("And sleeping for %ds until %s", delay, rendered_until)
		await asyncio.sleep(delay)

def _get_track():
	"""Get a track and load everything we need."""
	# TODO: Have proper async database calls (if we can do it without
	# massively breaking encapsulation); psycopg2 has an async mode, and
	# aiopg links that in with asyncio.
	nexttrack = database.get_track_to_play()
	dub2 = pydub.AudioSegment.from_mp3("audio/" + nexttrack.filename).set_channels(2)
	# NOTE: Calling amen with a filename invokes a second load from disk,
	# duplicating work done above. However, it will come straight from the
	# disk cache, so the only real duplication is the decode-from-MP3; and
	# timing tests show that this is a measurable but not overly costly
	# addition on top of the time to do the actual analysis. KISS.
	t2 = amen.audio.Audio("audio/" + nexttrack.filename)
	return nexttrack, t2, dub2

async def infinitely_glitch():
	try:
		nexttrack, t2, dub2 = _get_track()
		skip = 0.0
		while True:
			track = nexttrack; t1 = t2; dub1 = dub2
			nexttrack, t2, dub2 = _get_track()
			# Combine this into the next track.
			# 1) Analyze using amen
			#    t1 = amen.audio.Audio(track.filename)
			# 2) Locate the end of the effective last beat
			#    t1.timings['beats'][-10:-1][*].duration -> avg
			#    t1_end = t1.timings['beats'][-1].time + avg_duration
			# 3) Locate the first beat of the next track
			#    t2 = amen.audio.Audio(nexttrack.filename)
			#    t2_start = t2.timings['beats'][0].time
			# 4) Count back from the end of the last beat
			#    t1_end - t2_start
			# 5) Cross-fade from that point to t1_end to t2_start
			# Possibly do the cross-fade in two sections, as the times
			# won't be the same. They're the fade-in duration of t1 and
			# the fade-out duration of t2. Note that they depend on each
			# other, so they can't just be stored as-is (although the
			# beat positions and durations can).

			# Note on units:
			# The Timedelta that we find in timings['beats'] can provide us
			# with float total_seconds(), or with a .value in nanoseconds.
			# We later on will prefer milliseconds, though, so we rescale.
			# In this code, all variables store ms unless otherwise stated.
			t1b = t1.timings['beats']
			beat_ns = sum(b.duration.value for b in t1b[-1-LAST_BEAT_AVG : -1]) // LAST_BEAT_AVG
			t1_end = (t1b[-1].time.value + beat_ns) // 1000000
			t1_length = int(t1.duration * 1000)
			t2_start = t2.timings['beats'][1].time.value // 1000000
			# 1) Render t1 from skip up to (t1_end-t2_start) - the bulk of the track
			bulk = dub1[skip : t1_end - t2_start]
			track_list.append({
				"id": track.id,
				"start_time": rendered_until,
				"details": track.track_details,
			})
			await _render_output_audio(bulk, track.filename)
			# 2) Fade across t2_start ms - this will get us to the downbeat
			# 3) Fade across (t1_length-t1_end) ms - this nicely rounds out the last track
			# 4) Go get the next track, but skip the first (t2_start+t1_length-t1_end) ms
			skip = t2_start + t1_length - t1_end
			# Dumb fade mode. Doesn't actually fade, just overlays.
			fadeout1 = dub1[t1_end - t2_start : t1_end]
			fadein1 = dub2[:t2_start]
			fade1 = fadeout1.overlay(fadein1)
			fadeout2 = dub1[t1_end:]
			fadein2 = dub2[t2_start:skip]
			fade2 = fadeout2.overlay(fadein2)
			await _render_output_audio(fade1, "xfade 1")
			await _render_output_audio(fade2, "xfade 2")
	finally:
		logging.warn("Infinite Glitch coroutine terminating due to exception")
		ffmpeg.stdin.close()

# ------ Main renderer coroutine -------

async def ffmpeg():
	logging.debug("renderer started")
	global ffmpeg
	ffmpeg = await asyncio.create_subprocess_exec("ffmpeg", "-ac", "2", "-f", "s16le", "-i", "-", "-f", "mp3", "-",
		stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
	asyncio.ensure_future(infinitely_glitch())
	totdata = 0
	logging.debug("Waiting for data from ffmpeg...")
	chunk = b""
	try:
		while ffmpeg.returncode is None:
			data = await ffmpeg.stdout.read(4096)
			if not data: break
			totdata += len(data)
			chunk += data
			# logging.debug("Received %d bytes [%d]", totdata, len(data))
			if data.startswith(b"\xFF\xFB") and len(chunk) > 1024*1024:
				global position
				songs.append(chunk)
				logging.debug("Adding another song section [%d, %d bytes]", len(songs) + position, len(chunk))
				chunk = b""
				if len(songs) > 32:
					songs.pop(0)
					position += 1
	finally:
		if ffmpeg.returncode is None:
			logging.warn("Terminating FFMPEG due to renderer exception")
			ffmpeg.terminate()
	logging.warn("Main renderer coroutine terminating")

# ------ End of main renderer. Simpler stuff follows. :) -------

@route("/all.mp3")
async def moosic(req):
	logging.debug("/all.mp3 requested")
	resp = web.StreamResponse()
	resp.content_type = "audio/mpeg"
	await resp.prepare(req)
	pos = position
	while True:
		if pos - position >= len(songs):
			# Not enough content. This should only happen
			# on startup, so we just queue ourselves a bit.
			await asyncio.sleep(1)
			continue
		if pos < position:
			# The client is so far behind that we've dropped
			# the chunk that would have been next. There's
			# really not much we can do; disconnect.
			break
		logging.debug("songs %d, pos %d, position %d", len(songs), pos, position)
		resp.write(songs[pos - position])
		await resp.drain()
		pos += 1
	return resp

@route("/status.json")
async def info(req):
	logging.debug("/status.json requested")
	# TODO: Clean up the track list, ditching entries way in the past.
	# It doesn't need to be an ever-growing history.
	return web.json_response({"ts": time.time(), "tracks": track_list}, headers={"Access-Control-Allow-Origin": "*"})

def run(port=8889):
	asyncio.ensure_future(ffmpeg())
	web.run_app(app, port=port)

if __name__ == '__main__':
	run()
<commit_msg>renderer: Incorporate rendered_until in the status info<commit_after>from aiohttp import web
import amen.audio
import pydub
import time
import asyncio
import logging
import subprocess
from . import database

# To determine the "effective length" of the last beat, we
# average the last N beats prior to it. Higher numbers give
# smoother results but may have issues with a close-out rall.
LAST_BEAT_AVG = 10

app = web.Application()

songs = []
position = 0
track_list = []

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

# ------ Helper functions for infinitely_glitch() -------

# The rate-limiting sleep will wait until the clock catches up to this point.
# We start it "ten seconds ago" so we get a bit of buffer to start off.
rendered_until = time.time() - 10
ffmpeg = None # aio subprocess where we're compressing to MP3
async def _render_output_audio(seg, fn):
	logging.info("Sending %d bytes of data for %s secs of %s", len(seg.raw_data), seg.duration_seconds, fn)
	ffmpeg.stdin.write(seg.raw_data)
	await ffmpeg.stdin.drain()
	global rendered_until; rendered_until += seg.duration_seconds
	delay = rendered_until - time.time()
	if delay > 0:
		logging.debug("And sleeping for %ds until %s", delay, rendered_until)
		await asyncio.sleep(delay)

def _get_track():
	"""Get a track and load everything we need."""
	# TODO: Have proper async database calls (if we can do it without
	# massively breaking encapsulation); psycopg2 has an async mode, and
	# aiopg links that in with asyncio.
	nexttrack = database.get_track_to_play()
	dub2 = pydub.AudioSegment.from_mp3("audio/" + nexttrack.filename).set_channels(2)
	# NOTE: Calling amen with a filename invokes a second load from disk,
	# duplicating work done above. However, it will come straight from the
	# disk cache, so the only real duplication is the decode-from-MP3; and
	# timing tests show that this is a measurable but not overly costly
	# addition on top of the time to do the actual analysis. KISS.
	t2 = amen.audio.Audio("audio/" + nexttrack.filename)
	return nexttrack, t2, dub2

async def infinitely_glitch():
	try:
		nexttrack, t2, dub2 = _get_track()
		skip = 0.0
		while True:
			track = nexttrack; t1 = t2; dub1 = dub2
			nexttrack, t2, dub2 = _get_track()
			# Combine this into the next track.
			# 1) Analyze using amen
			#    t1 = amen.audio.Audio(track.filename)
			# 2) Locate the end of the effective last beat
			#    t1.timings['beats'][-10:-1][*].duration -> avg
			#    t1_end = t1.timings['beats'][-1].time + avg_duration
			# 3) Locate the first beat of the next track
			#    t2 = amen.audio.Audio(nexttrack.filename)
			#    t2_start = t2.timings['beats'][0].time
			# 4) Count back from the end of the last beat
			#    t1_end - t2_start
			# 5) Cross-fade from that point to t1_end to t2_start
			# Possibly do the cross-fade in two sections, as the times
			# won't be the same. They're the fade-in duration of t1 and
			# the fade-out duration of t2. Note that they depend on each
			# other, so they can't just be stored as-is (although the
			# beat positions and durations can).

			# Note on units:
			# The Timedelta that we find in timings['beats'] can provide us
			# with float total_seconds(), or with a .value in nanoseconds.
			# We later on will prefer milliseconds, though, so we rescale.
			# In this code, all variables store ms unless otherwise stated.
			t1b = t1.timings['beats']
			beat_ns = sum(b.duration.value for b in t1b[-1-LAST_BEAT_AVG : -1]) // LAST_BEAT_AVG
			t1_end = (t1b[-1].time.value + beat_ns) // 1000000
			t1_length = int(t1.duration * 1000)
			t2_start = t2.timings['beats'][1].time.value // 1000000
			# 1) Render t1 from skip up to (t1_end-t2_start) - the bulk of the track
			bulk = dub1[skip : t1_end - t2_start]
			track_list.append({
				"id": track.id,
				"start_time": rendered_until,
				"details": track.track_details,
			})
			await _render_output_audio(bulk, track.filename)
			# 2) Fade across t2_start ms - this will get us to the downbeat
			# 3) Fade across (t1_length-t1_end) ms - this nicely rounds out the last track
			# 4) Go get the next track, but skip the first (t2_start+t1_length-t1_end) ms
			skip = t2_start + t1_length - t1_end
			# Dumb fade mode. Doesn't actually fade, just overlays.
			fadeout1 = dub1[t1_end - t2_start : t1_end]
			fadein1 = dub2[:t2_start]
			fade1 = fadeout1.overlay(fadein1)
			fadeout2 = dub1[t1_end:]
			fadein2 = dub2[t2_start:skip]
			fade2 = fadeout2.overlay(fadein2)
			await _render_output_audio(fade1, "xfade 1")
			await _render_output_audio(fade2, "xfade 2")
	finally:
		logging.warn("Infinite Glitch coroutine terminating due to exception")
		ffmpeg.stdin.close()

# ------ Main renderer coroutine -------

async def ffmpeg():
	logging.debug("renderer started")
	global ffmpeg
	ffmpeg = await asyncio.create_subprocess_exec("ffmpeg", "-ac", "2", "-f", "s16le", "-i", "-", "-f", "mp3", "-",
		stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
	asyncio.ensure_future(infinitely_glitch())
	totdata = 0
	logging.debug("Waiting for data from ffmpeg...")
	chunk = b""
	try:
		while ffmpeg.returncode is None:
			data = await ffmpeg.stdout.read(4096)
			if not data: break
			totdata += len(data)
			chunk += data
			# logging.debug("Received %d bytes [%d]", totdata, len(data))
			if data.startswith(b"\xFF\xFB") and len(chunk) > 1024*1024:
				global position
				songs.append(chunk)
				logging.debug("Adding another song section [%d, %d bytes]", len(songs) + position, len(chunk))
				chunk = b""
				if len(songs) > 32:
					songs.pop(0)
					position += 1
	finally:
		if ffmpeg.returncode is None:
			logging.warn("Terminating FFMPEG due to renderer exception")
			ffmpeg.terminate()
	logging.warn("Main renderer coroutine terminating")

# ------ End of main renderer. Simpler stuff follows. :) -------

@route("/all.mp3")
async def moosic(req):
	logging.debug("/all.mp3 requested")
	resp = web.StreamResponse()
	resp.content_type = "audio/mpeg"
	await resp.prepare(req)
	pos = position
	while True:
		if pos - position >= len(songs):
			# Not enough content. This should only happen
			# on startup, so we just queue ourselves a bit.
			await asyncio.sleep(1)
			continue
		if pos < position:
			# The client is so far behind that we've dropped
			# the chunk that would have been next. There's
			# really not much we can do; disconnect.
			break
		logging.debug("songs %d, pos %d, position %d", len(songs), pos, position)
		resp.write(songs[pos - position])
		await resp.drain()
		pos += 1
	return resp

@route("/status.json")
async def info(req):
	logging.debug("/status.json requested")
	# TODO: Clean up the track list, ditching entries way in the past.
	# It doesn't need to be an ever-growing history.
	return web.json_response({
		"ts": time.time(), "render_time": rendered_until,
		"tracks": track_list
	}, headers={"Access-Control-Allow-Origin": "*"})

def run(port=8889):
	asyncio.ensure_future(ffmpeg())
	web.run_app(app, port=port)

if __name__ == '__main__':
	run()
<|endoftext|>


<commit_before>"""
                The Node Class
                --------------

  The basic building block for the linked list implementation is the node.
Each node object must hold at least two pieces of information. First, the node
must contain the list item itself. We will call this the data field of the node.
In addition, each node must hold a reference to the next node.
"""

class Node:
    def __init__(self, initdata):
        self.data = initdata
        self.next = None

    def getData(self):
        return self.data

    def getNext(self):
        return self.next

    def setData(self, newdata):
        self.data = newdata

    def setNext(self, newnext):
        self.next = newnext

"""
The Unordered List Class:
-------------------------

  The unordered list will be built from a collection of nodes, each linked to
the next by explicit references. As long as we know where to find the first
node, each item after that can be found by successively following the next
links. With this in mind, the UnorderedList class must maintain a reference to
the first node. Note that each list object will maintain a single reference to
the head of the list.

  The head of the list refers to the first node which contains the first item
of the list. In turn, that node holds a reference to the next node (the next
item) and so on. It is very important to note that the list class itself does
not contain any node objects. Instead it contains a single reference to only the first node in the linked structure.

  The isEmpty method simply checks to see if the head of the list is a reference
to None. The result of the boolean expression self.head==None will only be
true if there are no nodes in the linked list.

  So, how do we get items into our list? We need to implement the add method.
However, before we can do that, we need to address the important question of
where in the linked list to place the new item. Since this list is unordered,
the specific location of the new item with respect to the other items already
in the list is not important.The new item can go anywhere. With that in mind,
it makes sense to place the new item in the easiest location possible.

  Recall that the linked list structure provides us with only one entry point,
the head of the list. All of the other nodes can only be reached by accessing
the first node and then following next links. This means that the easiest place
to add the new node is right at the head, or beginning, of the list. In other
words, we will make the new item the first item of the list and the existing
items will need to be linked to this new first item so that they follow.

  The next methods that we will implement--length, search, and remove-- are all
based on a technique known as linked list traversal. Traversal refers to the
process of systematically visiting each node. To do this we use an external
reference that starts at the first node in the list. As we visit each node,
we move the reference the the next node by "traversing" the next reference.

  To implement the length method, we need to traverse the linked list and
keep a count of the number of nodes that occurred.

  Searching for a value in a linked list implementation of an unordered list
also uses the traversal technique. As we visit each node in the linked list we
will ask whether the data stored there matches the item we are looking for.

  The remove method requires two logical steps. First we need to traverse the
list looking for the item we want to remove. Once we find the item (recall that
we assume it is present), we must remove it. The first step is very similar to
search. Starting with an external reference set to the head of the list, we
traverse the links until we discover the item we are looking for. Since we
assume that item is present, we know that the iteration will stop before
current gets to None. This means that we can simply use the boolean found in
the condition.

  When found becomes True, current will be a reference to the node containing
the item to be removed. But how do we remove it? In order to the node containing
we need to modify the link in the previous node so that it referes to the node
that comes after current. Unfortunately, there is no way to go backward in the
linked list. Since current refers the the node ahead of the node where we would
like to make the change, it is too late to make the necessary modifcation.

  The solution to this dilemma is to use two external references as we traverse
down the linked list. current will behave just as it did before, marking the
current location of the traverse. The new reference , which we will call
previous, will always travel one node behind current. That way, when current
stops at the node to be removed, previous will be referring to the proper place
in the linked list for the modifcation.

  Once the searching step of the remove has been completed, we need to remove
the node from the linked list. There is a special case that needs to be
addressed. If the item to be removed happens to be the first item in the list,
then current will reference the first node in the linked list. This also means
that previous will be None. In this case, it is not previous but rather the
head of the list that needs to be changed.

  The remaining methods append, insert, and pop are left as exercises.
Remember that each of these must take into account whether the change is
taking place at the head of the list or someplace else. Also, insert, index,
and pop require that we name the posistions of the list. We will assume that
position names are integers starting with 0.
"""

class UnorderedList:
    def __init__(self):
        self.head = None

    def isEmpty(self):
        return self.head == None

    def add(self, item):
        temp = Node(item)
        temp.setNext(self.head)
        self.head = temp

    def length(self):
        current = self.head
        count = 0
        while current != None:
            count = count + 1
            current = current.getNext()

        return count

    def search(self, item):
        current = self.head
        found = False
        while current != None and not found:
            if current.getData() == item:
                found = True
            else:
                current = current.getNext()

        return found

    def remove(self, item):
        current = self.head
        previous = None
        found = False
        while not found and current != None:
            if current.getData() == item:
                found = True
            else:
                previous = current
                current = current.getNext()

        if found:
            if previous == None:
                self.head = current.getNext()
            else:
                previous.setNext(current.getNext())

    def pop(self, index = 0):
        current = self.head
        previous = None
        count = 0
        while count < index:
            previous = current
            current = current.getNext()
            count = count + 1

        if previous == None:
            pop = current.getData()
            self.head = current.getNext()
            return pop
        else:
            pop = current.getData()
            previous.setNext(current.getNext())
            return pop

    def index(self, item):
        current = self.head
        count = 0
        found = False
        while current != None and not found:
            if current.getData() == item:
                found = True
            else:
                current = current.getNext()
                count = count + 1

        if found:
            return count
        else:
            return found

    def append(self, item):
        current = self.head
        previous = None
        while current != None:
            previous = current
            current = current.getNext()

        if previous == None:
            tempt = Node(item)
            tempt.setNext(self.head)
            self.head = tempt
        else:
            tempt = Node(item)
            previous.setNext(tempt)

if __name__ == "__main__":
    unordered_list = UnorderedList()
    print("unordered_list.isEmpty():", unordered_list.isEmpty())
    unordered_list.append(9)
    unordered_list.add(10)
    unordered_list.add(11)
    unordered_list.add(12)
    unordered_list.add(13)
    unordered_list.append(14)
    print("unordered_list.append(9)")
    print("unordered_list.add(10)")
    print("unordered_list.add(11)")
    print("unordered_list.add(12)")
    print("unordered_list.add(13)")
    print("unordered_list.append(14)")
    print("unordered_list.length():", unordered_list.length())
    print("unordered_list.index(10):", unordered_list.index(10))
    print("unordered_list.index(11):", unordered_list.index(11))
    print("unordered_list.index(12):", unordered_list.index(12))
    print("unordered_list.index(13):", unordered_list.index(13))
    print("unordered_list.index(14):", unordered_list.index(14))
    print("unordered_list.pop(0):", unordered_list.pop(0))
    print("unordered_list.pop():", unordered_list.pop())
    print("unordered_list.pop(1):", unordered_list.pop(1))
    print("unordered_list.search(10):", unordered_list.search(10))
    print("unordered_list.search(1):", unordered_list.search(1))
    print("unordered_list.remove(10):", unordered_list.remove(10))
    print("unordered_list.length():", unordered_list.length())
    print("unordered_list.remove(2):", unordered_list.remove(2))
<commit_msg>Not done with the insert method. Finish it when I get home<commit_after>"""
                The Node Class
                --------------

  The basic building block for the linked list implementation is the node.
Each node object must hold at least two pieces of information. First, the node
must contain the list item itself. We will call this the data field of the node.
In addition, each node must hold a reference to the next node.
"""

class Node:
    def __init__(self, initdata):
        self.data = initdata
        self.next = None

    def getData(self):
        return self.data

    def getNext(self):
        return self.next

    def setData(self, newdata):
        self.data = newdata

    def setNext(self, newnext):
        self.next = newnext

"""
The Unordered List Class:
-------------------------

  The unordered list will be built from a collection of nodes, each linked to
the next by explicit references. As long as we know where to find the first
node, each item after that can be found by successively following the next
links. With this in mind, the UnorderedList class must maintain a reference to
the first node. Note that each list object will maintain a single reference to
the head of the list.

  The head of the list refers to the first node which contains the first item
of the list. In turn, that node holds a reference to the next node (the next
item) and so on. It is very important to note that the list class itself does
not contain any node objects. Instead it contains a single reference to only the first node in the linked structure.

  The isEmpty method simply checks to see if the head of the list is a reference
to None. The result of the boolean expression self.head==None will only be
true if there are no nodes in the linked list.

  So, how do we get items into our list? We need to implement the add method.
However, before we can do that, we need to address the important question of
where in the linked list to place the new item. Since this list is unordered,
the specific location of the new item with respect to the other items already
in the list is not important.The new item can go anywhere. With that in mind,
it makes sense to place the new item in the easiest location possible.

  Recall that the linked list structure provides us with only one entry point,
the head of the list. All of the other nodes can only be reached by accessing
the first node and then following next links. This means that the easiest place
to add the new node is right at the head, or beginning, of the list. In other
words, we will make the new item the first item of the list and the existing
items will need to be linked to this new first item so that they follow.

  The next methods that we will implement--length, search, and remove-- are all
based on a technique known as linked list traversal. Traversal refers to the
process of systematically visiting each node. To do this we use an external
reference that starts at the first node in the list. As we visit each node,
we move the reference the the next node by "traversing" the next reference.

  To implement the length method, we need to traverse the linked list and
keep a count of the number of nodes that occurred.

  Searching for a value in a linked list implementation of an unordered list
also uses the traversal technique. As we visit each node in the linked list we
will ask whether the data stored there matches the item we are looking for.

  The remove method requires two logical steps. First we need to traverse the
list looking for the item we want to remove. Once we find the item (recall that
we assume it is present), we must remove it. The first step is very similar to
search. Starting with an external reference set to the head of the list, we
traverse the links until we discover the item we are looking for. Since we
assume that item is present, we know that the iteration will stop before
current gets to None. This means that we can simply use the boolean found in
the condition.

  When found becomes True, current will be a reference to the node containing
the item to be removed. But how do we remove it? In order to the node containing
we need to modify the link in the previous node so that it referes to the node
that comes after current. Unfortunately, there is no way to go backward in the
linked list. Since current refers the the node ahead of the node where we would
like to make the change, it is too late to make the necessary modifcation.

  The solution to this dilemma is to use two external references as we traverse
down the linked list. current will behave just as it did before, marking the
current location of the traverse. The new reference , which we will call
previous, will always travel one node behind current. That way, when current
stops at the node to be removed, previous will be referring to the proper place
in the linked list for the modifcation.

  Once the searching step of the remove has been completed, we need to remove
the node from the linked list. There is a special case that needs to be
addressed. If the item to be removed happens to be the first item in the list,
then current will reference the first node in the linked list. This also means
that previous will be None. In this case, it is not previous but rather the
head of the list that needs to be changed.

  The remaining methods append, insert, and pop are left as exercises.
Remember that each of these must take into account whether the change is
taking place at the head of the list or someplace else. Also, insert, index,
and pop require that we name the posistions of the list. We will assume that
position names are integers starting with 0.
"""

class UnorderedList:
    def __init__(self):
        self.head = None

    def isEmpty(self):
        return self.head == None

    def add(self, item):
        temp = Node(item)
        temp.setNext(self.head)
        self.head = temp

    def length(self):
        current = self.head
        count = 0
        while current != None:
            count = count + 1
            current = current.getNext()

        return count

    def search(self, item):
        current = self.head
        found = False
        while current != None and not found:
            if current.getData() == item:
                found = True
            else:
                current = current.getNext()

        return found

    def remove(self, item):
        current = self.head
        previous = None
        found = False
        while not found and current != None:
            if current.getData() == item:
                found = True
            else:
                previous = current
                current = current.getNext()

        if found:
            if previous == None:
                self.head = current.getNext()
            else:
                previous.setNext(current.getNext())

    def pop(self, index = 0):
        current = self.head
        previous = None
        count = 0
        while count < index:
            previous = current
            current = current.getNext()
            count = count + 1

        if previous == None:
            pop = current.getData()
            self.head = current.getNext()
            return pop
        else:
            pop = current.getData()
            previous.setNext(current.getNext())
            return pop

    def index(self, item):
        current = self.head
        count = 0
        found = False
        while current != None and not found:
            if current.getData() == item:
                found = True
            else:
                current = current.getNext()
                count = count + 1

        if found:
            return count
        else:
            return found

    def insert(self, item, index):
        current = self.head
        previous = None
        count = 0
        while current != None and count < index:
            if count == index:
                tempt = Node(item)

    def append(self, item):
        current = self.head
        previous = None
        while current != None:
            previous = current
            current = current.getNext()

        if previous == None:
            tempt = Node(item)
            tempt.setNext(self.head)
            self.head = tempt
        else:
            tempt = Node(item)
            previous.setNext(tempt)

if __name__ == "__main__":
    unordered_list = UnorderedList()
    print("unordered_list.isEmpty():", unordered_list.isEmpty())
    unordered_list.append(9)
    unordered_list.add(10)
    unordered_list.add(11)
    unordered_list.add(12)
    unordered_list.add(13)
    unordered_list.append(14)
    print("unordered_list.append(9)")
    print("unordered_list.add(10)")
    print("unordered_list.add(11)")
    print("unordered_list.add(12)")
    print("unordered_list.add(13)")
    print("unordered_list.append(14)")
    print("unordered_list.length():", unordered_list.length())
    print("unordered_list.index(10):", unordered_list.index(10))
    print("unordered_list.index(11):", unordered_list.index(11))
    print("unordered_list.index(12):", unordered_list.index(12))
    print("unordered_list.index(13):", unordered_list.index(13))
    print("unordered_list.index(14):", unordered_list.index(14))
    print("unordered_list.pop(0):", unordered_list.pop(0))
    print("unordered_list.pop():", unordered_list.pop())
    print("unordered_list.pop(1):", unordered_list.pop(1))
    print("unordered_list.search(10):", unordered_list.search(10))
    print("unordered_list.search(1):", unordered_list.search(1))
    print("unordered_list.remove(10):", unordered_list.remove(10))
    print("unordered_list.length():", unordered_list.length())
    print("unordered_list.remove(2):", unordered_list.remove(2))
<|endoftext|>


<commit_before>from importlib import import_module
import warnings
from django.utils import importlib

class PillowtopConfigurationException(Exception):
    pass

def import_settings():
    try:
        from django.conf import settings
    except Exception, ex:
        # if we are not in a django context, then import local pillowsettings
        print "django import"
        print ex
        import pillowsettings as settings
    return settings

def import_pillows(instantiate=True):
    warnings.warn('pillowtop.utils.import_pillows deprecated, '
                  'please use pillowtop.get_all_pillows instead.',
                  DeprecationWarning)
    return get_all_pillows(instantiate=instantiate)

def import_pillow_string(full_class_str, instantiate=True):
    mod_path, pillow_class_name = full_class_str.rsplit('.', 1)
    try:
        mod = importlib.import_module(mod_path)
        pillowtop_class = getattr(mod, pillow_class_name)
        return pillowtop_class() if instantiate else pillowtop_class
    except (AttributeError, ImportError):
        raise ValueError("Could not find pillowtop class '%s'" % full_class_str)

def get_all_pillows(instantiate=True):
    settings = import_settings()

    pillowtops = []
    if hasattr(settings, 'PILLOWTOPS'):
        for k, v in settings.PILLOWTOPS.items():
            for full_str in v:
                pillowtop_class = import_pillow_string(full_str, instantiate=instantiate)
                pillowtops.append(pillowtop_class)

    return pillowtops

def get_pillow_by_name(pillow_class_name):
    all_pillows = get_all_pillows()
    for pillow in all_pillows:
        if pillow.__class__.__name__ == pillow_class_name:
            return pillow

def force_seq_int(seq):
    if seq is None:
        return None
    elif isinstance(seq, basestring):
        return int(seq.split('-')[0])
    else:
        assert isinstance(seq, int)
        return seq

def get_all_pillows_json():
    pillows = get_all_pillows()
    pillows_json = []
    for pillow in pillows:
        checkpoint = pillow.get_checkpoint()
        pillows_json.append({
            'name': pillow.__class__.__name__,
            'seq': force_seq_int(checkpoint.get('seq')),
            'old_seq': force_seq_int(checkpoint.get('old_seq')) or 0,
            'db_seq': force_seq_int(pillow.get_db_seq()),
        })
    return pillows_json
<commit_msg>add time since last checkpoint to pillows json<commit_after>from importlib import import_module
import warnings
from datetime import datetime
from dateutil.parser import parse
from django.utils import importlib
import pytz

class PillowtopConfigurationException(Exception):
    pass

def import_settings():
    try:
        from django.conf import settings
    except Exception, ex:
        # if we are not in a django context, then import local pillowsettings
        print "django import"
        print ex
        import pillowsettings as settings
    return settings

def import_pillows(instantiate=True):
    warnings.warn('pillowtop.utils.import_pillows deprecated, '
                  'please use pillowtop.get_all_pillows instead.',
                  DeprecationWarning)
    return get_all_pillows(instantiate=instantiate)

def import_pillow_string(full_class_str, instantiate=True):
    mod_path, pillow_class_name = full_class_str.rsplit('.', 1)
    try:
        mod = importlib.import_module(mod_path)
        pillowtop_class = getattr(mod, pillow_class_name)
        return pillowtop_class() if instantiate else pillowtop_class
    except (AttributeError, ImportError):
        raise ValueError("Could not find pillowtop class '%s'" % full_class_str)

def get_all_pillows(instantiate=True):
    settings = import_settings()

    pillowtops = []
    if hasattr(settings, 'PILLOWTOPS'):
        for k, v in settings.PILLOWTOPS.items():
            for full_str in v:
                pillowtop_class = import_pillow_string(full_str, instantiate=instantiate)
                pillowtops.append(pillowtop_class)

    return pillowtops

def get_pillow_by_name(pillow_class_name):
    all_pillows = get_all_pillows()
    for pillow in all_pillows:
        if pillow.__class__.__name__ == pillow_class_name:
            return pillow

def force_seq_int(seq):
    if seq is None:
        return None
    elif isinstance(seq, basestring):
        return int(seq.split('-')[0])
    else:
        assert isinstance(seq, int)
        return seq

def get_all_pillows_json():
    pillows = get_all_pillows()
    pillows_json = []
    for pillow in pillows:
        checkpoint = pillow.get_checkpoint()
        timestamp = checkpoint.get('timestamp')
        if timestamp:
            time_since_last = str(datetime.now(tz=pytz.UTC) - parse(timestamp))
            try:
                # remove microsecond portion
                time_since_last = time_since_last[0:time_since_last.index('.')]
            except ValueError:
                pass
        else:
            time_since_last = ''
        pillows_json.append({
            'name': pillow.__class__.__name__,
            'seq': force_seq_int(checkpoint.get('seq')),
            'old_seq': force_seq_int(checkpoint.get('old_seq')) or 0,
            'db_seq': force_seq_int(pillow.get_db_seq()),
            'time_since_last': time_since_last
        })
    return pillows_json
<|endoftext|>