<commit_before>from flask import render_template, request, flash, redirect, url_for
from casework import app
from .mint import Mint
from forms import RegistrationForm
import simplejson
import random

mint = Mint()

def generate_title_number():
    return 'TEST%d' % random.randint(1, 9999)

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/registration', methods=['GET','POST'])
def registration():

    form = RegistrationForm(request.form)
    property_frontend_url = '%s/%s' % (app.config['PROPERTY_FRONTEND_URL'], 'property')
    created = request.args.get('created', None)

    if  request.method == 'GET':
      #put the title number into the form's hidden field
      form.title_number.data = generate_title_number()

    if request.method == 'POST' and form.validate():
        mint_data = form_to_json(form)
        title_number = form['title_number'].data
        try:
            response = mint.post(title_number, mint_data)
            app.logger.info('Created title number %s at the mint url %s: status code %d'
                            % (title_number, response.url, response.status_code))

            return redirect('%s?created=%s' % (url_for('registration'), title_number))

        except RuntimeError as e:
            app.logger.error('Failed to register title %s: Error %s' % (title_number, e))
            flash('Creation of title with number %s failed' % title_number)

    return render_template('registration.html', form=form, property_frontend_url=property_frontend_url,
              title_number=form.title_number.data, created=created)

def form_to_json(form):
    data = simplejson.dumps({
      "title_number": form['title_number'].data,
      "proprietors":[
        {
          "first_name": form['first_name1'].data,
          "last_name": form['surname1'].data #can we sort names out please? not have last_name and surname2?
        },
        {
          "first_name": form['first_name2'].data,
          "last_name": form['surname2'].data
        }
      ],
      "property":{

        "address": {
          "house_number": form['house_number'].data,
          "road": form['road'].data,
          "town": form['town'].data,
          "postcode": form['postcode'].data
        },
        "tenure": form['property_tenure'].data,
        "class_of_title":  form['property_class'].data
      },
      "payment": {
        "price_paid": form['price_paid'].data,
        "titles":[
          form['title_number'].data
        ]
      }
    })
    return data

@app.errorhandler(404)
def page_not_found(error):
    return render_template('error.html', error = error), 404

@app.errorhandler(500)
def internal_server_error(error):
    return render_template('error.html', error = error), 500

#  Some useful headers to set to beef up the robustness of the app
# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
@app.after_request
def after_request(response):
    response.headers.add('Content-Security-Policy', "default-src 'self' 'unsafe-inline' data:")
    response.headers.add('X-Frame-Options', 'deny')
    response.headers.add('X-Content-Type-Options', 'nosniff')
    response.headers.add('X-XSS-Protection', '1; mode=block')
    return response
<commit_msg>Adding cdn to security policy<commit_after>from flask import render_template, request, flash, redirect, url_for
from casework import app
from .mint import Mint
from forms import RegistrationForm
import simplejson
import random

mint = Mint()

def generate_title_number():
    return 'TEST%d' % random.randint(1, 9999)

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/registration', methods=['GET','POST'])
def registration():

    form = RegistrationForm(request.form)
    property_frontend_url = '%s/%s' % (app.config['PROPERTY_FRONTEND_URL'], 'property')
    created = request.args.get('created', None)

    if  request.method == 'GET':
      #put the title number into the form's hidden field
      form.title_number.data = generate_title_number()

    if request.method == 'POST' and form.validate():
        mint_data = form_to_json(form)
        title_number = form['title_number'].data
        try:
            response = mint.post(title_number, mint_data)
            app.logger.info('Created title number %s at the mint url %s: status code %d'
                            % (title_number, response.url, response.status_code))

            return redirect('%s?created=%s' % (url_for('registration'), title_number))

        except RuntimeError as e:
            app.logger.error('Failed to register title %s: Error %s' % (title_number, e))
            flash('Creation of title with number %s failed' % title_number)

    return render_template('registration.html', form=form, property_frontend_url=property_frontend_url,
              title_number=form.title_number.data, created=created)

def form_to_json(form):
    data = simplejson.dumps({
      "title_number": form['title_number'].data,
      "proprietors":[
        {
          "first_name": form['first_name1'].data,
          "last_name": form['surname1'].data #can we sort names out please? not have last_name and surname2?
        },
        {
          "first_name": form['first_name2'].data,
          "last_name": form['surname2'].data
        }
      ],
      "property":{

        "address": {
          "house_number": form['house_number'].data,
          "road": form['road'].data,
          "town": form['town'].data,
          "postcode": form['postcode'].data
        },
        "tenure": form['property_tenure'].data,
        "class_of_title":  form['property_class'].data
      },
      "payment": {
        "price_paid": form['price_paid'].data,
        "titles":[
          form['title_number'].data
        ]
      }
    })
    return data

@app.errorhandler(404)
def page_not_found(error):
    return render_template('error.html', error = error), 404

@app.errorhandler(500)
def internal_server_error(error):
    return render_template('error.html', error = error), 500

#  Some useful headers to set to beef up the robustness of the app
# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
@app.after_request
def after_request(response):
    response.headers.add('Content-Security-Policy', "default-src 'self' 'unsafe-inline' data: http://maxcdn.bootstrapcdn.com")
    response.headers.add('X-Frame-Options', 'deny')
    response.headers.add('X-Content-Type-Options', 'nosniff')
    response.headers.add('X-XSS-Protection', '1; mode=block')
    return response
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import numpy as np
import scipy.spatial.distance as spdist

import dask.array as da

import pytest

import dask_distance

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_bool_dist_err(funcname, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_bool_dist(funcname, seed, size, chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, (size,), dtype=bool)
    a_v = np.random.randint(0, 2, (size,), dtype=bool)

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    a_r = sp_func(a_u, a_v)
    d_r = da_func(d_u, d_v)

    assert np.array(d_r) == a_r

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((3, 10), (1, 5), (3, 10), (1, 5)),
    ]
)
def test_2d_bool_dist(funcname, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    a_v = np.random.randint(0, 2, v_shape, dtype=bool)

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    da_func = getattr(dask_distance, funcname)

    a_r = spdist.cdist(a_u, a_v, funcname)
    d_r = da_func(d_u, d_v)

    assert np.allclose(np.array(d_r), a_r)
<commit_msg>Adjust one bool array's shape in the 2D tests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import numpy as np
import scipy.spatial.distance as spdist

import dask.array as da

import pytest

import dask_distance

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_bool_dist_err(funcname, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_bool_dist(funcname, seed, size, chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, (size,), dtype=bool)
    a_v = np.random.randint(0, 2, (size,), dtype=bool)

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    a_r = sp_func(a_u, a_v)
    d_r = da_func(d_u, d_v)

    assert np.array(d_r) == a_r

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((2, 10), (1, 5), (3, 10), (1, 5)),
    ]
)
def test_2d_bool_dist(funcname, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    a_v = np.random.randint(0, 2, v_shape, dtype=bool)

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    da_func = getattr(dask_distance, funcname)

    a_r = spdist.cdist(a_u, a_v, funcname)
    d_r = da_func(d_u, d_v)

    assert np.allclose(np.array(d_r), a_r)
<|endoftext|>


<commit_before>import spark
import motion
from bot import process_command
from config import config
from flask import Flask
from flask import request
from flask import jsonify
from threading import Thread
import time
import sys
app = Flask(__name__)

def on_motion_detected():
    print("motion detected!")

def run_motion_detection():
    motion.detector_on(on_motion_detected)

def run_flask_server():
    app.run(host='0.0.0.0', port=8181)

@app.route("/", methods=["post"])
def index():
    # Parse request
    webhook_req = request.get_json()
    message = spark.get_message(message_id=webhook_req['data']['id'], bearer=config["bearer"])

    if message["personEmail"] != config["bot_email"]:
        res = process_command(message["command"], message)
        if res["response_required"]:
            spark.send_message(message["roomId"], res["data"], config["bearer"])

    return jsonify({})

if __name__ == "__main__":
    motion_thread = Thread(target = run_motion_detection)
    motion_thread.daemon = True
    motion_thread.start()
    app.run(host='0.0.0.0', port=8080)
<commit_msg>Photo by notification<commit_after>import spark
import motion
import commands.camera as camera
import commands.event as event
from bot import process_command
from config import config
from flask import Flask
from flask import request
from flask import jsonify
from threading import Thread
import time
import sys
app = Flask(__name__)

def on_motion_detected():
    room_data = event.get_subscribers("security")
    print(room_data)
    if len(room_data):
        photo_data = camera.take_photo()
        for room in room_data:
            spark.send_message(room["room_id"], photo_data["data"], config["bearer"])

def run_motion_detection():
    motion.detector_on(on_motion_detected)

def run_flask_server():
    app.run(host='0.0.0.0', port=8181)

@app.route("/", methods=["post"])
def index():
    # Parse request
    webhook_req = request.get_json()
    message = spark.get_message(message_id=webhook_req['data']['id'], bearer=config["bearer"])

    if message["personEmail"] != config["bot_email"]:
        res = process_command(message["command"], message)
        if res["response_required"]:
            spark.send_message(message["roomId"], res["data"], config["bearer"])

    return jsonify({})

if __name__ == "__main__":
    motion_thread = Thread(target = run_motion_detection)
    motion_thread.daemon = True
    motion_thread.start()
    app.run(host='0.0.0.0', port=8080)
<|endoftext|>


<commit_before>from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
from scrapy.utils.url import is_url
from scrapy.spider import spiders
from scrapy.http import Request
from scrapy import log

class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] <domain|url> ..."

    def short_desc(self):
        return "Start crawling a domain or URL"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline")
        parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls")
        parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", help="don't follow links (for use with URLs only)")
        parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for starting to crawl the given url")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        if opts.nopipeline:
            settings.overrides['ITEM_PIPELINES'] = []

        if opts.restrict:
            settings.overrides['RESTRICT_TO_URLS'] = args

        if opts.nofollow:
            settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

    def run(self, args, opts):
        if opts.callback:
            requests = []
            for a in args:
                if is_url(a):
                    spider = spiders.fromurl(a)
                    urls = [a]
                else:
                    spider = spiders.fromdomain(a)
                    urls = spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls]

                if spider:
                    if hasattr(spider, opts.callback):
                        requests.extend(Request(url=url, callback=getattr(spider, opts.callback)) for url in urls)
                    else:
                        log.msg('Callback %s doesnt exist in spider %s' % (opts.callback, spider.domain_name), log.ERROR)
                else:
                    log.msg('Cannot find spider for %s' % a, log.ERROR)

            if requests:
                args = requests
            else:
                log.msg('Cannot create any requests with the provided arguments', log.ERROR)
                return

        scrapymanager.runonce(*args)
<commit_msg>some line wrapping at 80 cols<commit_after>from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
from scrapy.utils.url import is_url
from scrapy.spider import spiders
from scrapy.http import Request
from scrapy import log

class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] <domain|url> ..."

    def short_desc(self):
        return "Start crawling a domain or URL"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--nopipeline", dest="nopipeline", action="store_true", \
            help="disable scraped item pipeline")
        parser.add_option("--restrict", dest="restrict", action="store_true", \
            help="restrict crawling only to the given urls")
        parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
            help="don't follow links (for use with URLs only)")
        parser.add_option("-c", "--callback", dest="callback", action="store", \
            help="use the provided callback for starting to crawl the given url")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        if opts.nopipeline:
            settings.overrides['ITEM_PIPELINES'] = []

        if opts.restrict:
            settings.overrides['RESTRICT_TO_URLS'] = args

        if opts.nofollow:
            settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

    def run(self, args, opts):
        if opts.callback:
            requests = []
            for a in args:
                if is_url(a):
                    spider = spiders.fromurl(a)
                    urls = [a]
                else:
                    spider = spiders.fromdomain(a)
                    urls = spider.start_urls if hasattr(spider.start_urls, \
                        '__iter__') else [spider.start_urls]

                if spider:
                    if hasattr(spider, opts.callback):
                        requests.extend(Request(url=url, callback=getattr(spider, \
                            opts.callback)) for url in urls)
                    else:
                        log.msg('Callback %s doesnt exist in spider %s' % \
                            (opts.callback, spider.domain_name), log.ERROR)
                else:
                    log.msg('Cannot find spider for %s' % a, log.ERROR)

            if requests:
                args = requests
            else:
                log.msg('Cannot create any requests with the provided arguments', \
                    log.ERROR)
                return

        scrapymanager.runonce(*args)
<|endoftext|>


<commit_before>import django
from django.shortcuts import redirect

try:
    from django.urls import resolve
except ImportError:
    from django.core.urlresolvers import resolve

from django.utils.deprecation import MiddlewareMixin

from .actions import customers, subscriptions
from .conf import settings

class ActiveSubscriptionMiddleware(MiddlewareMixin):

    def process_request(self, request):
        is_authenticated = request.user.is_authenticated
        if django.VERSION < (1, 10):
            is_authenticated = is_authenticated()

        if is_authenticated and not request.user.is_staff:
            url_name = resolve(request.path).url_name
            if url_name not in settings.PINAX_STRIPE_SUBSCRIPTION_REQUIRED_EXCEPTION_URLS:
                customer = customers.get_customer_for_user(request.user)
                if not subscriptions.has_active_subscription(customer):
                    return redirect(
                        settings.PINAX_STRIPE_SUBSCRIPTION_REQUIRED_REDIRECT
                    )
<commit_msg>fallback if django is < 1.10<commit_after>import django
from django.shortcuts import redirect

try:
    from django.urls import resolve
except ImportError:
    from django.core.urlresolvers import resolve

try:
    from django.utils.deprecation import MiddlewareMixin as MixinorObject
except ImportError:
    MixinorObject = object

from .actions import customers, subscriptions
from .conf import settings

class ActiveSubscriptionMiddleware(MixinorObject):

    def process_request(self, request):
        is_authenticated = request.user.is_authenticated
        if django.VERSION < (1, 10):
            is_authenticated = is_authenticated()

        if is_authenticated and not request.user.is_staff:
            url_name = resolve(request.path).url_name
            if url_name not in settings.PINAX_STRIPE_SUBSCRIPTION_REQUIRED_EXCEPTION_URLS:
                customer = customers.get_customer_for_user(request.user)
                if not subscriptions.has_active_subscription(customer):
                    return redirect(
                        settings.PINAX_STRIPE_SUBSCRIPTION_REQUIRED_REDIRECT
                    )
<|endoftext|>


<commit_before>#! /usr/bin/env python3

import os   # deleting the db
import sys  # output redirection
import html # unescape naugty html codes
import urllib.request
import sqlite3  # db access
import datetime
import logging
# graphical interface imports
import tkinter as tk
import tkinter.ttk as ttk

# installed modules
from bs4 import BeautifulSoup

# local files
from fixer import ALL_FIX_DICT, VOCAB_FIX_DICT
from greektools import greek_simplify, greek_to_ascii

operoneUrl = 'http://localhost/web/phayax/agt/res/operone/altspr/wadinhalt.html'
operoneBaseUrl = "http://localhost/web/phayax/agt/res/operone/altspr/"

DB_NAME = 'agt.sqlite'

#operoneUrl = 'http://www.operone.de/altspr/wadinhalt.html'
#operoneBaseUrl = "http://www.operone.de/altspr/"

unknown_set = {""}
problematic_list = {""}

class StdoutRedirector:
    '''
    Eine Klasse um Output zu einem Text-Widget umzuleiten.
    Wird hauptsächlich für stdout verwendet.
    stderr hat eine eigene Klasse mit Highlighting.
    '''
    def __init__(self, textitem):
        # Die Klasse braucht ein Handle auf das Text-Widget,
        # um dorthin zu schreiben
        self.texthandle = textitem

    def write(self, string):
        self.texthandle.insert(tk.END, string)

    def flush(self):
        self.texthandle.update()

class App(tk.Tk):

    def __init__(self, cursor):
        tk.Tk.__init__(self)
        self.initialize()
        self.dbHandle = cursor

    def initialize(self):
        self.title('Operone -> Database Parser')
        self.grid()
        self.minsize(800,600)

        self.grid_columnconfigure(1, weight=1)

        self.infoText = tk.Label(self, text="Huhu")
        self.infoText.grid(row=0, column=1, padx=5, pady=5)

        self.startButton = tk.Button(self, text='Start Parsing', command=self.startParsing)
        self.startButton.grid(row = 2, column = 1, padx=5, pady=5)
        self.quitButton = tk.Button(self, text='Close',fg='red', command=self.quit)
        self.quitButton.grid(row=3, column = 1, padx=5, pady=5)

        self.progress = tk.DoubleVar()
        self.progress.set(0)

        self.progressBar = ttk.Progressbar(self, orient="horizontal", maximum=266, length=500, mode="determinate", variable=self.progress)
        self.progressBar.grid(row = 1, column = 1, sticky="WE", padx=5, pady=5)

        #self.barstarted = False
        yScrollbar = tk.Scrollbar(self, orient='vertical')
        yScrollbar.grid(row=4, column=2, sticky='NSE')
        xScrollbar = tk.Scrollbar(self, orient='horizontal')
        xScrollbar.grid(row=5, column=1, sticky='WE')

        self.output = tk.Text(self,wrap=tk.NONE, height = 4, width=50, yscrollcommand=yScrollbar.set, xscrollcommand=xScrollbar.set)
        self.output.grid(row=4, column=1, sticky='NSWE', padx=5, pady=5)
        yScrollbar.config(command=self.output.yview)
        xScrollbar.config(command=self.output.xview)

        self.grid_rowconfigure(4, weight=1)

        self.entriesParsed = 0

        # backup stdout
        self.origstdout = sys.stdout
        # reroute stdout to application text widget
        sys.stdout = StdoutRedirector(self.output)

    def quit(self):
        # restore original stdout
        sys.stdout = self.origstdout
        self.destroy()

    def startParsing(self):
        startTime = datetime.datetime.now()
        self.progress.set(0)
        pages = parseIndex(self.dbHandle)
        for idx, page in enumerate(pages):
            self.infoText.configure(text='Parsing page {}'.format(page))
            self.entriesParsed += parsePage(self.dbHandle, page, idx)
            self.progress.set(self.progress.get() + 1.0)
            self.progressBar.update()
        endTime = datetime.datetime.now()
        timepassed = (endTime - startTime).total_seconds()
        self.infoText.configure(text='Finished Parsing: {} entries processed in {:.1f} seconds'.format(self.entriesParsed, timepassed))

def getListLinks(soup):
    """
    Takes all "li > a" tags and returns them.
    The correct selector is probably "li > a:firstChild".
    """
    # get all li's
    lis = soup.find_all('li')
    # get all a's that are inside the captured li's.
    lis = [el.find('a') for el in lis if len(el.find_all('a')) > 0]

    return lis

def fixBadHtml(source):
    """
    Takes bad html lists and puts closing tags on the end of each line
    with a list tag and no closing tag.
    """
    # process each line on it's own
    source = source.split('\n')

    newSource = []

    for line in source:
        line = line.strip()
        # check if its a relevant line
        # we know that only entries are li's
        if '
' in line:
            if not '
' in line:
                line += '
'

        newSource.append(line)

    newSource = '\n'.join(newSource)
    return newSource

def createTables(connection):

    # Create a table as an index of all pages.
    connection.execute('CREATE TABLE IF NOT EXISTS pageindex(idx INTEGER PRIMARY KEY, page TEXT, link TEXT);')
    logging.info("[Created table 'pageindex']")

    # Create a table of the content of all pages.
    #connection.execute('CREATE TABLE IF NOT EXISTS pagecontent(pagenum INTEGER, word TEXT, alternateWords TEXT, translation TEXT);')
    connection.execute('CREATE TABLE IF NOT EXISTS pagecontent(roughword TEXT, preciseword TEXT, greek TEXT, alternategreek TEXT, translation TEXT);')
    logging.info("[Created table 'pagecontent']")

def parseExceptions(line):
    if type(line) != str:
        raise TypeError('type str expected. found {}'.format(type(line)))

    # detect if there was a missing  tag that got wrongly autocorrected by beautifulsoup
    # this is done by checking if there is anything between the closing li tag and the closing span tag.
    closeSpan = line.find('')
    closeSpan += len('')
    closeLi = line.find('
')
    if len(line[closeSpan:closeLi].strip()) == 0:
        logging.info('-----------------------------------------------------------------------')
        logging.info('[Warning] Possible problem in the following line.:\n\t{}'.format(line))
        # remove the wrongly placed closing span
        line = line[:line.find('')] + line[line.find('
'):]
        # add the closing span at the first space after the greek word
        # this is not failproof but the problem will be reported and can be checked
        insertIdx = line.find('')
        # note the trailing space - there is "always"(ha!) a space between the tag and the word.
        # so we are setting the index right before the beginning of the word
        # if not it is placed in the word which is just fine
        insertIdx += len(' ')
        # look for the first whitespace starting from the previously
        # determined index.
        insertIdx = line.find(' ', insertIdx)
        # insert the closing span tag plus a comma for separation since
        # the later part expects the word to be divided from the translation by a comma
        # if we introduce a comma too much that is not a problem since the later part will
        # filter out commas.
        line = line[:insertIdx] + ', ' + line[insertIdx:]
        logging.info('\t\t Attempting to fix the mentioned problem. Please check:\n\t\t{}'.format(line))

    # apply fix dict for whole lines
    for element in ALL_FIX_DICT:
        line = line.replace(element, ALL_FIX_DICT[element])

    return line

def parseIndex(c):

    # get source of index page
    source = urllib.request.urlopen(urllib.request.Request(operoneUrl)).read() #(chapterUrl, headers={'User-Agent': AGENT_NAME + ' Browser'})).read()
    source = str(source)
    source = html.unescape(source)

    soup = BeautifulSoup(source, 'html.parser')

    pages = getListLinks(soup)

    for idx, entry in enumerate(pages):
        page = entry.get_text(strip=True)
        link = entry.get('href')
        params = (idx + 1, page, link,)
        c.execute('INSERT INTO pageindex VALUES(?, ?, ?);', params)

    return pages

def parsePage(c, page, idx):
    """
    Parses a single page of the operone dictionary:
    Gets the page source:
    fixes problematic html tags(not closed or wrongly placed mostly)
    for every entry on the page:
        runs parseExceptions on the line
        takes the different parts of the line.
        simplifies and copys the transformed versions.
        writes the entry to the database
    """
    page = urllib.request.urlopen(urllib.request.Request(operoneBaseUrl + page.get('href'))).read()
    page = page.decode('ISO-8859-1') # encoding of the operone pages
    page = html.unescape(page)
    #page = str(page) # cast to string (from stream)
    #page = html.unescape(page) # unescape greek letters
    page = fixBadHtml(page)

    # this automatically adds closing span tags at the end of a line if none are present
    pageSoup = BeautifulSoup(page, 'html.parser')
    lis = pageSoup.find_all('li')

    for element in lis:
        #print(element)
        # ok, we are getting additional translations or variations that belong to the word
        # for now we will be ignoring them by using only the first child.

        correctedLine = parseExceptions(str(element))
        element = BeautifulSoup(correctedLine, 'html.parser')

        # get_text can strip whitespaces but since we need the comma stripped as well
        # it makes more sense to put both into one context.
        # vocab is the raw string of the entry
        vocab = element.findChild().findChild().get_text().strip(', ')

        # apply fix dict for vocab part
        for entry in VOCAB_FIX_DICT:
            vocab = vocab.replace(entry, VOCAB_FIX_DICT[entry])

        # versions is a list of the different lookup words of the entry
        versions = [version.strip() for version in vocab.split(',')]
        # main is the first version - we will just assume that this is what we want ...
        main = versions[0]
        # alternate are all other versions concatenated by commas.
        alternate = ",".join(versions[1:])
        # start index behind the first span. used to separate
        # the lookup word from the translation since there can be
        # greek letters and tags in the translation (otherwise we could just the text with recursive=False to eliminate text in tags)
        tlStartIndex = str(element).find('')
        tlStartIndex += len('')

        # ok, so here we take off the first part of the entry which
        # contains the greek word. Then we feed the remaining string into
        # a new BeautifulSoup instance and strip remaining tags in the translation
        # with the get_text() method.
        subText = BeautifulSoup((str(element))[tlStartIndex:], 'html.parser')
        translation = str(subText.get_text()).strip()
        #pageNum = idx
        rough = greek_to_ascii(greek_simplify(main), False)
        precise = greek_to_ascii(greek_simplify(main), True)
        #c.execute('INSERT INTO pagecontent VALUES(?, ?, ?, ?, ?)',(rough, precise, main, alternate, translation,))
        #print(translation)

    return len(lis)

def main():

    logging.basicConfig(filename='dbgen.log', filemode='w', level=logging.WARNING)
    # remove old db file
    if os.path.exists(DB_NAME):
        os.remove(DB_NAME)
        print('[Removed old database]')

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # sqlite setup
    conn = sqlite3.connect(DB_NAME)
    print('[Created new database]')
    c = conn.cursor()

    createTables(c)

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # tkinter related
    app = App(c)
    app.mainloop()

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # sqlite cleanup
    conn.commit()
    print('[Changes commited]')
    conn.close()
    print('[Database closed]')

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # Debugging operone data
    logging.debug(unknown_set)
    probs = []
    logging.debug('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    logging.debug('problematic elements that could not easily be converted')
    for el in problematic_list:
        logging.debug(el)

if __name__ == '__main__':
    main()

<commit_msg> reactivated the writing to db<commit_after>#! /usr/bin/env python3

import os   # deleting the db
import sys  # output redirection
import html # unescape naugty html codes
import urllib.request
import sqlite3  # db access
import datetime
import logging
# graphical interface imports
import tkinter as tk
import tkinter.ttk as ttk

# installed modules
from bs4 import BeautifulSoup

# local files
from fixer import ALL_FIX_DICT, VOCAB_FIX_DICT
from greektools import greek_simplify, greek_to_ascii

operoneUrl = 'http://localhost/web/phayax/agt/res/operone/altspr/wadinhalt.html'
operoneBaseUrl = "http://localhost/web/phayax/agt/res/operone/altspr/"

DB_NAME = 'agt.sqlite'

#operoneUrl = 'http://www.operone.de/altspr/wadinhalt.html'
#operoneBaseUrl = "http://www.operone.de/altspr/"

unknown_set = {""}
problematic_list = {""}

class StdoutRedirector:
    '''
    Eine Klasse um Output zu einem Text-Widget umzuleiten.
    Wird hauptsächlich für stdout verwendet.
    stderr hat eine eigene Klasse mit Highlighting.
    '''
    def __init__(self, textitem):
        # Die Klasse braucht ein Handle auf das Text-Widget,
        # um dorthin zu schreiben
        self.texthandle = textitem

    def write(self, string):
        self.texthandle.insert(tk.END, string)

    def flush(self):
        self.texthandle.update()

class App(tk.Tk):

    def __init__(self, cursor):
        tk.Tk.__init__(self)
        self.initialize()
        self.dbHandle = cursor

    def initialize(self):
        self.title('Operone -> Database Parser')
        self.grid()
        self.minsize(800,600)

        self.grid_columnconfigure(1, weight=1)

        self.infoText = tk.Label(self, text="Huhu")
        self.infoText.grid(row=0, column=1, padx=5, pady=5)

        self.startButton = tk.Button(self, text='Start Parsing', command=self.startParsing)
        self.startButton.grid(row = 2, column = 1, padx=5, pady=5)
        self.quitButton = tk.Button(self, text='Close',fg='red', command=self.quit)
        self.quitButton.grid(row=3, column = 1, padx=5, pady=5)

        self.progress = tk.DoubleVar()
        self.progress.set(0)

        self.progressBar = ttk.Progressbar(self, orient="horizontal", maximum=266, length=500, mode="determinate", variable=self.progress)
        self.progressBar.grid(row = 1, column = 1, sticky="WE", padx=5, pady=5)

        #self.barstarted = False
        yScrollbar = tk.Scrollbar(self, orient='vertical')
        yScrollbar.grid(row=4, column=2, sticky='NSE')
        xScrollbar = tk.Scrollbar(self, orient='horizontal')
        xScrollbar.grid(row=5, column=1, sticky='WE')

        self.output = tk.Text(self,wrap=tk.NONE, height = 4, width=50, yscrollcommand=yScrollbar.set, xscrollcommand=xScrollbar.set)
        self.output.grid(row=4, column=1, sticky='NSWE', padx=5, pady=5)
        yScrollbar.config(command=self.output.yview)
        xScrollbar.config(command=self.output.xview)

        self.grid_rowconfigure(4, weight=1)

        self.entriesParsed = 0

        # backup stdout
        self.origstdout = sys.stdout
        # reroute stdout to application text widget
        sys.stdout = StdoutRedirector(self.output)

    def quit(self):
        # restore original stdout
        sys.stdout = self.origstdout
        self.destroy()

    def startParsing(self):
        startTime = datetime.datetime.now()
        self.progress.set(0)
        pages = parseIndex(self.dbHandle)
        for idx, page in enumerate(pages):
            self.infoText.configure(text='Parsing page {}'.format(page))
            self.entriesParsed += parsePage(self.dbHandle, page, idx)
            self.progress.set(self.progress.get() + 1.0)
            self.progressBar.update()
        endTime = datetime.datetime.now()
        timepassed = (endTime - startTime).total_seconds()
        self.infoText.configure(text='Finished Parsing: {} entries processed in {:.1f} seconds'.format(self.entriesParsed, timepassed))

def getListLinks(soup):
    """
    Takes all "li > a" tags and returns them.
    The correct selector is probably "li > a:firstChild".
    """
    # get all li's
    lis = soup.find_all('li')
    # get all a's that are inside the captured li's.
    lis = [el.find('a') for el in lis if len(el.find_all('a')) > 0]

    return lis

def fixBadHtml(source):
    """
    Takes bad html lists and puts closing tags on the end of each line
    with a list tag and no closing tag.
    """
    # process each line on it's own
    source = source.split('\n')

    newSource = []

    for line in source:
        line = line.strip()
        # check if its a relevant line
        # we know that only entries are li's
        if '
' in line:
            if not '
' in line:
                line += '
'

        newSource.append(line)

    newSource = '\n'.join(newSource)
    return newSource

def createTables(connection):

    # Create a table as an index of all pages.
    connection.execute('CREATE TABLE IF NOT EXISTS pageindex(idx INTEGER PRIMARY KEY, page TEXT, link TEXT);')
    logging.info("[Created table 'pageindex']")

    # Create a table of the content of all pages.
    #connection.execute('CREATE TABLE IF NOT EXISTS pagecontent(pagenum INTEGER, word TEXT, alternateWords TEXT, translation TEXT);')
    connection.execute('CREATE TABLE IF NOT EXISTS pagecontent(roughword TEXT, preciseword TEXT, greek TEXT, alternategreek TEXT, translation TEXT);')
    logging.info("[Created table 'pagecontent']")

def parseExceptions(line):
    if type(line) != str:
        raise TypeError('type str expected. found {}'.format(type(line)))

    # detect if there was a missing  tag that got wrongly autocorrected by beautifulsoup
    # this is done by checking if there is anything between the closing li tag and the closing span tag.
    closeSpan = line.find('')
    closeSpan += len('')
    closeLi = line.find('
')
    if len(line[closeSpan:closeLi].strip()) == 0:
        logging.info('-----------------------------------------------------------------------')
        logging.info('[Warning] Possible problem in the following line.:\n\t{}'.format(line))
        # remove the wrongly placed closing span
        line = line[:line.find('')] + line[line.find('
'):]
        # add the closing span at the first space after the greek word
        # this is not failproof but the problem will be reported and can be checked
        insertIdx = line.find('')
        # note the trailing space - there is "always"(ha!) a space between the tag and the word.
        # so we are setting the index right before the beginning of the word
        # if not it is placed in the word which is just fine
        insertIdx += len(' ')
        # look for the first whitespace starting from the previously
        # determined index.
        insertIdx = line.find(' ', insertIdx)
        # insert the closing span tag plus a comma for separation since
        # the later part expects the word to be divided from the translation by a comma
        # if we introduce a comma too much that is not a problem since the later part will
        # filter out commas.
        line = line[:insertIdx] + ', ' + line[insertIdx:]
        logging.info('\t\t Attempting to fix the mentioned problem. Please check:\n\t\t{}'.format(line))

    # apply fix dict for whole lines
    for element in ALL_FIX_DICT:
        line = line.replace(element, ALL_FIX_DICT[element])

    return line

def parseIndex(c):

    # get source of index page
    source = urllib.request.urlopen(urllib.request.Request(operoneUrl)).read() #(chapterUrl, headers={'User-Agent': AGENT_NAME + ' Browser'})).read()
    source = str(source)
    source = html.unescape(source)

    soup = BeautifulSoup(source, 'html.parser')

    pages = getListLinks(soup)

    for idx, entry in enumerate(pages):
        page = entry.get_text(strip=True)
        link = entry.get('href')
        params = (idx + 1, page, link,)
        c.execute('INSERT INTO pageindex VALUES(?, ?, ?);', params)

    return pages

def parsePage(c, page, idx):
    """
    Parses a single page of the operone dictionary:
    Gets the page source:
    fixes problematic html tags(not closed or wrongly placed mostly)
    for every entry on the page:
        runs parseExceptions on the line
        takes the different parts of the line.
        simplifies and copys the transformed versions.
        writes the entry to the database
    """
    page = urllib.request.urlopen(urllib.request.Request(operoneBaseUrl + page.get('href'))).read()
    page = page.decode('ISO-8859-1') # encoding of the operone pages
    page = html.unescape(page)
    #page = str(page) # cast to string (from stream)
    #page = html.unescape(page) # unescape greek letters
    page = fixBadHtml(page)

    # this automatically adds closing span tags at the end of a line if none are present
    pageSoup = BeautifulSoup(page, 'html.parser')
    lis = pageSoup.find_all('li')

    for element in lis:
        #print(element)
        # ok, we are getting additional translations or variations that belong to the word
        # for now we will be ignoring them by using only the first child.

        correctedLine = parseExceptions(str(element))
        element = BeautifulSoup(correctedLine, 'html.parser')

        # get_text can strip whitespaces but since we need the comma stripped as well
        # it makes more sense to put both into one context.
        # vocab is the raw string of the entry
        vocab = element.findChild().findChild().get_text().strip(', ')

        # apply fix dict for vocab part
        for entry in VOCAB_FIX_DICT:
            vocab = vocab.replace(entry, VOCAB_FIX_DICT[entry])

        # versions is a list of the different lookup words of the entry
        versions = [version.strip() for version in vocab.split(',')]
        # main is the first version - we will just assume that this is what we want ...
        main = versions[0]
        # alternate are all other versions concatenated by commas.
        alternate = ",".join(versions[1:])
        # start index behind the first span. used to separate
        # the lookup word from the translation since there can be
        # greek letters and tags in the translation (otherwise we could just the text with recursive=False to eliminate text in tags)
        tlStartIndex = str(element).find('')
        tlStartIndex += len('')

        # ok, so here we take off the first part of the entry which
        # contains the greek word. Then we feed the remaining string into
        # a new BeautifulSoup instance and strip remaining tags in the translation
        # with the get_text() method.
        subText = BeautifulSoup((str(element))[tlStartIndex:], 'html.parser')
        translation = str(subText.get_text()).strip()
        #pageNum = idx
        rough = greek_to_ascii(greek_simplify(main), False)
        precise = greek_to_ascii(greek_simplify(main), True)
        c.execute('INSERT INTO pagecontent VALUES(?, ?, ?, ?, ?)',(rough, precise, main, alternate, translation,))
        #print(translation)

    return len(lis)

def main():

    logging.basicConfig(filename='dbgen.log', filemode='w', level=logging.WARNING)
    # remove old db file
    if os.path.exists(DB_NAME):
        os.remove(DB_NAME)
        print('[Removed old database]')

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # sqlite setup
    conn = sqlite3.connect(DB_NAME)
    print('[Created new database]')
    c = conn.cursor()

    createTables(c)

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # tkinter related
    app = App(c)
    app.mainloop()

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # sqlite cleanup
    conn.commit()
    print('[Changes commited]')
    conn.close()
    print('[Database closed]')

    # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
    # Debugging operone data
    logging.debug(unknown_set)
    probs = []
    logging.debug('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    logging.debug('problematic elements that could not easily be converted')
    for el in problematic_list:
        logging.debug(el)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from calvin.runtime.north.calvin_token import Token
from calvin.runtime.north.plugins.port.queue.common import QueueFull, QueueEmpty, COMMIT_RESPONSE
from calvin.runtime.north.plugins.port import DISCONNECT
from calvin.utilities import calvinlogger

_log = calvinlogger.get_logger(__name__)

class CollectBase(object):

    """
    A queue with fanin support, does not handle token order between connections
    only within a connection.
    """

    def __init__(self, port_properties, peer_port_properties):
        super(CollectBase, self).__init__()
        # Set default queue length to 4 if not specified
        length = port_properties.get('queue_length', 4)
        # Compensate length for FIFO having an unused slot
        length += 1
        # Each peer have it's own FIFO
        self.fifo = {}
        self.nbr_peers = port_properties.get('nbr_peers', 1)
        self.N = length
        # Peers ordered by id
        self.writers = []
        # NOTE: For simplicity, modulo operation is only used in fifo access,
        #       all read and write positions are monotonousy increasing
        self.write_pos = {}
        self.read_pos = {}
        self.tentative_read_pos = {}
        self.tags = {}
        self.exhausted_tokens = {}
        self.termination = {}  # dictionary of tuples (termination type, got exhaust tokens bool)

    def __str__(self):
        fifo = "\n".join([str(k) + ": " + ", ".join(map(lambda x: str(x), self.fifo[k])) for k in self.fifo.keys()])
        return "Tokens: %s\nw:%s, r:%s, tr:%s" % (fifo, self.write_pos, self.read_pos, self.tentative_read_pos)

    def _state(self, remap=None):
        if remap is None:
            state = {
                'queuetype': self._type,
                'fifo': {p: [t.encode() for t in tokens] for p, tokens in self.fifo.items()},
                'N': self.N,
                'writers': self.writers,
                'write_pos': self.write_pos,
                'read_pos': self.read_pos,
                'tentative_read_pos': self.tentative_read_pos,
                'tags': self.tags
            }
        else:
            state = {
                'queuetype': self._type,
                'fifo': {remap[p] if p in remap else p: [Token(0).encode() for t in tokens] for p, tokens in self.fifo.items()},
                'N': self.N,
                'writers': sorted([remap[pid] if pid in remap else pid for pid in self.writers]),
                'write_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.write_pos.keys()},
                'read_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.read_pos.keys()},
                'tentative_read_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.tentative_read_pos.keys()},
                # TODO Need unique tags, how should these be created
                'tags': {remap[pid] if pid in remap else pid: tag for pid, tag in self.tags.items()}
            }
        return state

    def _set_state(self, state):
        self._type = state.get('queuetype')
        self.fifo = {p: [Token.decode(t) for t in tokens] for p, tokens in state['fifo'].items()}
        self.N = state['N']
        self.writers = state['writers']
        self.write_pos = state['write_pos']
        self.read_pos = state['read_pos']
        self.tentative_read_pos = state['tentative_read_pos']
        self.tags = state.get("tags", {})

    @property
    def queue_type(self):
        return self._type

    def add_writer(self, writer, properties):
        if not isinstance(writer, basestring):
            raise Exception('Not a string: %s' % writer)
        if writer not in self.writers:
            self.read_pos[writer] = 0
            self.write_pos[writer] = 0
            self.tentative_read_pos[writer] = 0
            self.fifo.setdefault(writer, [Token(0)] * self.N)
            self.writers.append(writer)
            self.writers.sort()
        if len(self.writers) > self.nbr_peers:
            _log.debug("ADD_WRITER %s" % writer)
            self.nbr_peers = len(self.writers)

        self.tags[writer] = properties.get("tag", writer)

    def remove_writer(self, writer):
        if not isinstance(writer, basestring):
            raise Exception('Not a string: %s' % writer)
        _log.debug("remove_writer %s %s" % (self.reader if hasattr(self, 'reader') else "--", writer))
        del self.read_pos[writer]
        del self.tentative_read_pos[writer]
        del self.write_pos[writer]
        del self.fifo[writer]
        del self.tags[writer]
        self.writers.remove(writer)
        self.nbr_peers -= 1

    def add_reader(self, reader, properties):
        # Only used for logging
        self.reader = reader

    def remove_reader(self, reader):
        pass

    def is_exhausting(self, peer_id=None):
        if peer_id is None:
            return bool(self.termination)
        else:
            return peer_id in self.termination

    def exhaust(self, peer_id, terminate):
        # We can't do anything until we consumed the last token
        self.termination[peer_id] = (terminate, False)
        _log.debug("exhaust %s %s %s" % (self._type, peer_id, DISCONNECT.reverse_mapping[terminate]))
        return []

    def any_outstanding_exhaustion_tokens(self):
        # Between having asked actor to exhaust and receiving exhaustion tokens we don't want to assume that
        # the exhaustion is done.
        return any([not t[1] for t in self.termination.values()])

    def set_exhausted_tokens(self, tokens):
        _log.debug("set_exhausted_tokens %s %s new:%s existing:%s writers:%s" % (
            self.reader if hasattr(self, 'reader') else "--", self._type, tokens,
            self.exhausted_tokens, self.writers))
        # Extend lists of current exhaust tokens, not replace (with likely empty list).
        # This is useful when a disconnect happens from both directions or other reasons
        self.exhausted_tokens.update({k: self.exhausted_tokens.get(k, []) + v   for k, v in tokens.items()})
        for peer_id in tokens.keys():
            self.termination[peer_id] = (self.termination[peer_id][0], True)
        remove = []
        for peer_id, exhausted_tokens in self.exhausted_tokens.items():
            if peer_id not in self.writers:
                # Make sure only writers in the exhausted_tokens
                remove.append(peer_id)
                continue
            if self._transfer_exhaust_tokens(peer_id, exhausted_tokens):
                remove.append(peer_id)
        for peer_id in remove:
            _log.debug("set_exhausted_tokens remove %s %s" % (peer_id, self.exhausted_tokens[peer_id]))
            del self.exhausted_tokens[peer_id]
        # Remove any terminated queues if empty
        for peer_id in tokens.keys():
            if peer_id not in self.writers:
                continue
            if (self.write_pos[peer_id] == self.read_pos[peer_id] and
                self.termination[peer_id][0] in [DISCONNECT.EXHAUST_PEER_RECV, DISCONNECT.EXHAUST_INPORT]):
                self.remove_writer(peer_id)
                del self.termination[peer_id]
        #fifos={w: [f[i%self.N] for i in range(self.tentative_read_pos[w], self.write_pos[w])]
        #        for w, f in self.fifo.items()}
        #_log.debug("set_exhausted_tokens2 %s %s left:%s queue: %s" %
        #           (self.reader if hasattr(self, 'reader') else "--", self._type, self.exhausted_tokens, fifos))
        return self.nbr_peers

    def _transfer_exhaust_tokens(self, peer_id, exhausted_tokens):
        # exhausted tokens are in sequence order, but could contain tokens already in queue
        for pos, token in exhausted_tokens[:]:
            if not self.slots_available(1, peer_id):
                break
            r = self.com_write(token, peer_id, pos)
            _log.debug("write exhausted tokens on %s, %s: (%d, %s) %s" % (
                peer_id, self.reader if hasattr(self, 'reader') else "--", pos, str(token), COMMIT_RESPONSE.reverse_mapping[r]))
            # This is a token that now is in the queue, was in the queue or is invalid, for all cases remove it
            exhausted_tokens.pop(0)
        return not bool(exhausted_tokens)

    def get_peers(self):
        return self.writers

    def write(self, data, metadata):
        if not self.slots_available(1, metadata):
            raise QueueFull()
        # Write token in peer's FIFO
        write_pos = self.write_pos[metadata]
        self.fifo[metadata][write_pos % self.N] = data
        self.write_pos[metadata] = write_pos + 1
        return True

    def slots_available(self, length, metadata):
        if metadata not in self.writers:
            raise Exception("No writer %s in %s" % (metadata, self.writers))
        return length < self.N and (self.write_pos[metadata] - self.read_pos[metadata]) < (self.N - length)

    def tokens_available(self, length, metadata):
        raise NotImplementedError("Sub-class must override")

    #
    # Reading is done tentatively until committed
    # The commit and cancel are only used in action firings not communication.
    # It is always all peeked tokens that are commited or canceled.
    #
    def peek(self, metadata):
        raise NotImplementedError("Sub-class must override")

    def commit(self, metadata):
        for writer in self.writers:
            self.read_pos[writer] = self.tentative_read_pos[writer]
        # Transfer in exhausted tokens when possible
        remove = []
        #_log.debug("commit collect exhausted_tokens %s" % self.exhausted_tokens)
        for peer_id, exhausted_tokens in self.exhausted_tokens.items():
            if self._transfer_exhaust_tokens(peer_id, exhausted_tokens):
                remove.append(peer_id)
        for peer_id in remove:
            del self.exhausted_tokens[peer_id]
        # When a terminated queue is fully consumed remove it
        remove = []
        for peer_id, termination in self.termination.items():
            if (self.write_pos[peer_id] == self.read_pos[peer_id] and
                termination[0] in [DISCONNECT.EXHAUST_PEER_RECV, DISCONNECT.EXHAUST_INPORT] and
                termination[1]):
                remove.append(peer_id)
        for peer_id in remove:
            #_log.debug("commit remove")
            self.remove_writer(peer_id)
            del self.termination[peer_id]
        return bool(remove)

    def cancel(self, metadata):
        for writer in self.writers:
            self.tentative_read_pos[writer] = self.read_pos[writer]

    #
    # Queue operations used by communication which utilize a sequence number
    #

    def com_write(self, data, metadata, sequence_nbr):
        write_pos = self.write_pos[metadata]
        if sequence_nbr == write_pos:
            self.write(data, metadata)
            return COMMIT_RESPONSE.handled
        elif sequence_nbr < write_pos:
            return COMMIT_RESPONSE.unhandled
        else:
            return COMMIT_RESPONSE.invalid

    def com_peek(self, metadata):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_commit(self, reader, sequence_nbr):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_cancel(self, reader, sequence_nbr):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_is_committed(self, reader):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

<commit_msg>collect_base: unused import<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from calvin.runtime.north.calvin_token import Token
from calvin.runtime.north.plugins.port.queue.common import QueueFull, COMMIT_RESPONSE
from calvin.runtime.north.plugins.port import DISCONNECT
from calvin.utilities import calvinlogger

_log = calvinlogger.get_logger(__name__)

class CollectBase(object):

    """
    A queue with fanin support, does not handle token order between connections
    only within a connection.
    """

    def __init__(self, port_properties, peer_port_properties):
        super(CollectBase, self).__init__()
        # Set default queue length to 4 if not specified
        length = port_properties.get('queue_length', 4)
        # Compensate length for FIFO having an unused slot
        length += 1
        # Each peer have it's own FIFO
        self.fifo = {}
        self.nbr_peers = port_properties.get('nbr_peers', 1)
        self.N = length
        # Peers ordered by id
        self.writers = []
        # NOTE: For simplicity, modulo operation is only used in fifo access,
        #       all read and write positions are monotonousy increasing
        self.write_pos = {}
        self.read_pos = {}
        self.tentative_read_pos = {}
        self.tags = {}
        self.exhausted_tokens = {}
        self.termination = {}  # dictionary of tuples (termination type, got exhaust tokens bool)

    def __str__(self):
        fifo = "\n".join([str(k) + ": " + ", ".join(map(lambda x: str(x), self.fifo[k])) for k in self.fifo.keys()])
        return "Tokens: %s\nw:%s, r:%s, tr:%s" % (fifo, self.write_pos, self.read_pos, self.tentative_read_pos)

    def _state(self, remap=None):
        if remap is None:
            state = {
                'queuetype': self._type,
                'fifo': {p: [t.encode() for t in tokens] for p, tokens in self.fifo.items()},
                'N': self.N,
                'writers': self.writers,
                'write_pos': self.write_pos,
                'read_pos': self.read_pos,
                'tentative_read_pos': self.tentative_read_pos,
                'tags': self.tags
            }
        else:
            state = {
                'queuetype': self._type,
                'fifo': {remap[p] if p in remap else p: [Token(0).encode() for t in tokens] for p, tokens in self.fifo.items()},
                'N': self.N,
                'writers': sorted([remap[pid] if pid in remap else pid for pid in self.writers]),
                'write_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.write_pos.keys()},
                'read_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.read_pos.keys()},
                'tentative_read_pos': {remap[pid] if pid in remap else pid: 0 for pid in self.tentative_read_pos.keys()},
                # TODO Need unique tags, how should these be created
                'tags': {remap[pid] if pid in remap else pid: tag for pid, tag in self.tags.items()}
            }
        return state

    def _set_state(self, state):
        self._type = state.get('queuetype')
        self.fifo = {p: [Token.decode(t) for t in tokens] for p, tokens in state['fifo'].items()}
        self.N = state['N']
        self.writers = state['writers']
        self.write_pos = state['write_pos']
        self.read_pos = state['read_pos']
        self.tentative_read_pos = state['tentative_read_pos']
        self.tags = state.get("tags", {})

    @property
    def queue_type(self):
        return self._type

    def add_writer(self, writer, properties):
        if not isinstance(writer, basestring):
            raise Exception('Not a string: %s' % writer)
        if writer not in self.writers:
            self.read_pos[writer] = 0
            self.write_pos[writer] = 0
            self.tentative_read_pos[writer] = 0
            self.fifo.setdefault(writer, [Token(0)] * self.N)
            self.writers.append(writer)
            self.writers.sort()
        if len(self.writers) > self.nbr_peers:
            _log.debug("ADD_WRITER %s" % writer)
            self.nbr_peers = len(self.writers)

        self.tags[writer] = properties.get("tag", writer)

    def remove_writer(self, writer):
        if not isinstance(writer, basestring):
            raise Exception('Not a string: %s' % writer)
        _log.debug("remove_writer %s %s" % (self.reader if hasattr(self, 'reader') else "--", writer))
        del self.read_pos[writer]
        del self.tentative_read_pos[writer]
        del self.write_pos[writer]
        del self.fifo[writer]
        del self.tags[writer]
        self.writers.remove(writer)
        self.nbr_peers -= 1

    def add_reader(self, reader, properties):
        # Only used for logging
        self.reader = reader

    def remove_reader(self, reader):
        pass

    def is_exhausting(self, peer_id=None):
        if peer_id is None:
            return bool(self.termination)
        else:
            return peer_id in self.termination

    def exhaust(self, peer_id, terminate):
        # We can't do anything until we consumed the last token
        self.termination[peer_id] = (terminate, False)
        _log.debug("exhaust %s %s %s" % (self._type, peer_id, DISCONNECT.reverse_mapping[terminate]))
        return []

    def any_outstanding_exhaustion_tokens(self):
        # Between having asked actor to exhaust and receiving exhaustion tokens we don't want to assume that
        # the exhaustion is done.
        return any([not t[1] for t in self.termination.values()])

    def set_exhausted_tokens(self, tokens):
        _log.debug("set_exhausted_tokens %s %s new:%s existing:%s writers:%s" % (
            self.reader if hasattr(self, 'reader') else "--", self._type, tokens,
            self.exhausted_tokens, self.writers))
        # Extend lists of current exhaust tokens, not replace (with likely empty list).
        # This is useful when a disconnect happens from both directions or other reasons
        self.exhausted_tokens.update({k: self.exhausted_tokens.get(k, []) + v   for k, v in tokens.items()})
        for peer_id in tokens.keys():
            self.termination[peer_id] = (self.termination[peer_id][0], True)
        remove = []
        for peer_id, exhausted_tokens in self.exhausted_tokens.items():
            if peer_id not in self.writers:
                # Make sure only writers in the exhausted_tokens
                remove.append(peer_id)
                continue
            if self._transfer_exhaust_tokens(peer_id, exhausted_tokens):
                remove.append(peer_id)
        for peer_id in remove:
            _log.debug("set_exhausted_tokens remove %s %s" % (peer_id, self.exhausted_tokens[peer_id]))
            del self.exhausted_tokens[peer_id]
        # Remove any terminated queues if empty
        for peer_id in tokens.keys():
            if peer_id not in self.writers:
                continue
            if (self.write_pos[peer_id] == self.read_pos[peer_id] and
                self.termination[peer_id][0] in [DISCONNECT.EXHAUST_PEER_RECV, DISCONNECT.EXHAUST_INPORT]):
                self.remove_writer(peer_id)
                del self.termination[peer_id]
        #fifos={w: [f[i%self.N] for i in range(self.tentative_read_pos[w], self.write_pos[w])]
        #        for w, f in self.fifo.items()}
        #_log.debug("set_exhausted_tokens2 %s %s left:%s queue: %s" %
        #           (self.reader if hasattr(self, 'reader') else "--", self._type, self.exhausted_tokens, fifos))
        return self.nbr_peers

    def _transfer_exhaust_tokens(self, peer_id, exhausted_tokens):
        # exhausted tokens are in sequence order, but could contain tokens already in queue
        for pos, token in exhausted_tokens[:]:
            if not self.slots_available(1, peer_id):
                break
            r = self.com_write(token, peer_id, pos)
            _log.debug("write exhausted tokens on %s, %s: (%d, %s) %s" % (
                peer_id, self.reader if hasattr(self, 'reader') else "--", pos, str(token), COMMIT_RESPONSE.reverse_mapping[r]))
            # This is a token that now is in the queue, was in the queue or is invalid, for all cases remove it
            exhausted_tokens.pop(0)
        return not bool(exhausted_tokens)

    def get_peers(self):
        return self.writers

    def write(self, data, metadata):
        if not self.slots_available(1, metadata):
            raise QueueFull()
        # Write token in peer's FIFO
        write_pos = self.write_pos[metadata]
        self.fifo[metadata][write_pos % self.N] = data
        self.write_pos[metadata] = write_pos + 1
        return True

    def slots_available(self, length, metadata):
        if metadata not in self.writers:
            raise Exception("No writer %s in %s" % (metadata, self.writers))
        return length < self.N and (self.write_pos[metadata] - self.read_pos[metadata]) < (self.N - length)

    def tokens_available(self, length, metadata):
        raise NotImplementedError("Sub-class must override")

    #
    # Reading is done tentatively until committed
    # The commit and cancel are only used in action firings not communication.
    # It is always all peeked tokens that are commited or canceled.
    #
    def peek(self, metadata):
        raise NotImplementedError("Sub-class must override")

    def commit(self, metadata):
        for writer in self.writers:
            self.read_pos[writer] = self.tentative_read_pos[writer]
        # Transfer in exhausted tokens when possible
        remove = []
        #_log.debug("commit collect exhausted_tokens %s" % self.exhausted_tokens)
        for peer_id, exhausted_tokens in self.exhausted_tokens.items():
            if self._transfer_exhaust_tokens(peer_id, exhausted_tokens):
                remove.append(peer_id)
        for peer_id in remove:
            del self.exhausted_tokens[peer_id]
        # When a terminated queue is fully consumed remove it
        remove = []
        for peer_id, termination in self.termination.items():
            if (self.write_pos[peer_id] == self.read_pos[peer_id] and
                termination[0] in [DISCONNECT.EXHAUST_PEER_RECV, DISCONNECT.EXHAUST_INPORT] and
                termination[1]):
                remove.append(peer_id)
        for peer_id in remove:
            #_log.debug("commit remove")
            self.remove_writer(peer_id)
            del self.termination[peer_id]
        return bool(remove)

    def cancel(self, metadata):
        for writer in self.writers:
            self.tentative_read_pos[writer] = self.read_pos[writer]

    #
    # Queue operations used by communication which utilize a sequence number
    #

    def com_write(self, data, metadata, sequence_nbr):
        write_pos = self.write_pos[metadata]
        if sequence_nbr == write_pos:
            self.write(data, metadata)
            return COMMIT_RESPONSE.handled
        elif sequence_nbr < write_pos:
            return COMMIT_RESPONSE.unhandled
        else:
            return COMMIT_RESPONSE.invalid

    def com_peek(self, metadata):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_commit(self, reader, sequence_nbr):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_cancel(self, reader, sequence_nbr):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

    def com_is_committed(self, reader):
        raise NotImplementedError("The unordered fanin queue should not be used on an outport")

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2006-2018  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia_test import *
from math import *
from essentia import array as esarr
import numpy as np

class TestHumDetector(TestCase):

    def testZero(self):
        self.assertEqualVector(HumDetector()(esarr(np.zeros(512)))[1], esarr([]))

    def testOnes(self):
        self.assertEqualVector(HumDetector()(esarr(np.ones(512)))[1], esarr([]))

    def testInvalidParam(self):
        self.assertConfigureFails(HumDetector(), {'sampleRate': -1})
        self.assertConfigureFails(HumDetector(), {'frameSize': 0})
        self.assertConfigureFails(HumDetector(), {'hopSize': 0})
        self.assertConfigureFails(HumDetector(), {'Q0': -1})
        self.assertConfigureFails(HumDetector(), {'Q1': 2})
        self.assertConfigureFails(HumDetector(), {'minimumFrequency': 0})
        self.assertConfigureFails(HumDetector(), {'timeWindow': 0})

    def testSyntheticHum(self):
        #  this test adds an artificial humming tone of 50 Hz
        import numpy as np
        filename = join(testdata.audio_dir, 'recorded/Vivaldi_Sonata_5_II_Allegro.wav')
        audio = MonoLoader(filename=filename)()

        nSamples = len(audio)

        time = np.linspace(0, nSamples / 44100., nSamples)

        freq = 50.

        hum = np.sin(2 * np.pi * freq * time )

        hum *= db2amp(-54.) # empirically found to be the audible limit

        _, f, _, _, _ = HumDetector()(np.array(audio + hum , dtype=np.float32))

        self.assertAlmostEqualVector(f, [freq], 1e1)

    def testARProcess(self):
        # This test assess the capability of the algorithm to
        # detect a low frequency humming modeled as an autoregresive
        # process. The idea is that this kind of signal is probably
        # closer to hummings found on real scenarios.
        PI = np.pi
        time = 30. # s
        fs = 44100. # fs
        freq = 100 # Hz
        nSamples = int(time * fs)

        attempts = 3
        for i in range(attempts):
            try:
                noise = np.array(np.random.randn(nSamples), dtype=np.float32)

                v1 = [1, -1 * np.exp(1j * 2 * PI * freq / fs)]
                v2 = [1, -1 * np.exp(-1j * 2 * PI * freq / fs)]

                a = np.array(np.convolve(np.real(v1), np.real(v2)), dtype=np.float32)
                b = 1 - np.sum(abs(a))

                hum = IIR(denominator=a, numerator=[1.])(noise)
                hum /= np.max(hum)

                signal = hum + noise
                signal /= np.max(signal)

                _, estimated_freqs, _, _, _, = HumDetector()(signal)

                self.assertAlmostEqualVector([estimated_freqs[0]], [freq], 1e2)
            except IndexError:
                print 'testARProcess failed. This test is based on random signals so it can fail sometimes. {}Attempt {}/{}'.format('It will be repeated. ' if i < 2 else '', i+1, attempts)
                continue
            if i > 0:
                print 'testARProcess passed on the attempt {}'.format(i+1)
            break

suite = allTests(TestHumDetector)

if __name__ == '__main__':
    TextTestRunner(verbosity=2).run(suite)
<commit_msg>HumDetector: fixed ARProcess test<commit_after>#!/usr/bin/env python

# Copyright (C) 2006-2018  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia_test import *
from math import *
from essentia import array as esarr
import numpy as np

class TestHumDetector(TestCase):

    def testZero(self):
        self.assertEqualVector(HumDetector()(esarr(np.zeros(512)))[1], esarr([]))

    def testOnes(self):
        self.assertEqualVector(HumDetector()(esarr(np.ones(512)))[1], esarr([]))

    def testInvalidParam(self):
        self.assertConfigureFails(HumDetector(), {'sampleRate': -1})
        self.assertConfigureFails(HumDetector(), {'frameSize': 0})
        self.assertConfigureFails(HumDetector(), {'hopSize': 0})
        self.assertConfigureFails(HumDetector(), {'Q0': -1})
        self.assertConfigureFails(HumDetector(), {'Q1': 2})
        self.assertConfigureFails(HumDetector(), {'minimumFrequency': 0})
        self.assertConfigureFails(HumDetector(), {'timeWindow': 0})

    def testSyntheticHum(self):
        #  this test adds an artificial humming tone of 50 Hz
        import numpy as np
        filename = join(testdata.audio_dir, 'recorded/Vivaldi_Sonata_5_II_Allegro.wav')
        audio = MonoLoader(filename=filename)()

        nSamples = len(audio)

        time = np.linspace(0, nSamples / 44100., nSamples)

        freq = 50.

        hum = np.sin(2 * np.pi * freq * time )

        hum *= db2amp(-54.) # empirically found to be the audible limit

        _, f, _, _, _ = HumDetector()(np.array(audio + hum , dtype=np.float32))

        self.assertAlmostEqualVector(f, [freq], 1e1)

    def testARProcess(self):
        # This test assess the capability of the algorithm to
        # detect a low frequency humming modeled as an autoregresive
        # process. The idea is that this kind of signal is probably
        # closer to hummings found on real scenarios.
        PI = np.pi
        time = 30. # s
        fs = 44100. # fs
        freq = 100 # Hz
        nSamples = int(time * fs)

        attempts = 3
        for i in range(attempts):
            try:
                noise = np.array(np.random.randn(nSamples), dtype=np.float32)

                v1 = [1, -1 * np.exp(1j * 2 * PI * freq / fs)]
                v2 = [1, -1 * np.exp(-1j * 2 * PI * freq / fs)]

                a = np.array(np.convolve(np.real(v1), np.real(v2)), dtype=np.float32)
                b = 1 - np.sum(abs(a))

                hum = IIR(denominator=a, numerator=[1.])(noise)
                hum /= np.max(hum)

                signal = hum + noise
                signal /= np.max(signal)

                _, estimated_freqs, _, _, _, = HumDetector(detectionThreshold=2.)(signal)

                self.assertAlmostEqualVector([estimated_freqs[0]], [freq], 1e2)
            except IndexError:
                if i == attempts - 1:
                    self.assertAlmostEqualVector([estimated_freqs[0]], [freq], 1e2)
                print 'testARProcess failed. This test is based on random signals so it can fail sometimes. {}Attempt {}/{}'.format('It will be repeated. ' if i < 2 else '', i+1, attempts)
                continue
            if i > 0:
                print 'testARProcess passed on the attempt {}'.format(i+1)
            break

suite = allTests(TestHumDetector)

if __name__ == '__main__':
    TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from collections.abc import Sequence
from tempfile import mkstemp
import os
import unittest

from ..tabular_data import TabularData, TabularDataFromFilePath
from ...exceptions import BaseError

class BaseTabularDataTest:

    def set_header (self, data, value = True):
        """Set the header to the given value (or to True)."""
        data.header = value

    def assert_header (self, data, value):
        """Get the data's header and assert its equality."""
        self.assertEqual(data.header, value)

    def assert_rows (self, data, value):
        """Get the data's row count and assert its equality."""
        self.assertEqual(data.rows, value)

    def assert_cols (self, data, value):
        """Get the data's column count and assert its equality."""
        self.assertEqual(data.cols, value)

    def data_from_str (self, content):
        """Shortcut for writing content to a file and opening it."""
        self.write_file(content)
        return self.init_tabular_data()

    def test_bad_bytes (self):
        for i in b"\x81\x8d\x8f\x90\x9d":
            self.write_file(i.to_bytes(1, "big"))
            with self.assertRaises(TabularData.CantDecodeEncoding):
                data = self.init_tabular_data()

    def test_bad_control_characters (self):
        ranges = (
            (0x00, 0x09), # skip HT, LF
            (0x0b, 0x0d), # skip CR
            (0x0e, 0x20), # skip [ -~] aka visible characters
            (0x7f, 0xa0), # not worrying about non-ASCII controls
        )

        for low, high in ranges:
            for i in range(low, high):
                self.write_file("something{}something\n".format(chr(i)))
                regex = "0x{:02x}".format(i)
                with self.assertRaisesRegex(TabularData.InvalidControls, regex):
                    data = self.init_tabular_data()

    def test_error_on_stupid_newlines (self):
        self.write_file("first line\nsecond line\r\nthird line\rfourth")

        with self.assertRaises(TabularData.InconsistentNewlines):
            data = self.init_tabular_data()

    def test_error_on_LF_and_CRLF (self):
        self.write_file("first line\nsecond line\r\nthird line")

        with self.assertRaises(TabularData.InconsistentNewlines):
            data = self.init_tabular_data()

    def test_error_on_inconsistent_columns (self):
        self.write_file("shipment\tvolume\n1234567\t8974326\t7843927")

        with self.assertRaises(TabularData.InconsistentColumnCounts):
            data = self.init_tabular_data()

    def set_to_3x3_grid (self):
        return self.data_from_str("\n".join((
                "first\tsecond\tthird",
                "matthew\talexander\tlachance",
                "un (french for one)\tdeux\ttrois")))

    def set_to_2x4_grid (self):
        return self.data_from_str("\n".join((
                "first\tsecond",
                "matthew\tlachance",
                "flibbity flobbity\tfloop",
                "un\tdeux")))

    def test_len_3x3 (self):
        data = self.set_to_3x3_grid()
        self.assertEqual(len(data), 3)
        self.assert_rows(data, 3)
        self.assert_cols(data, 3)

    def test_len_2x4 (self):
        data = self.set_to_2x4_grid()
        self.assertEqual(len(data), 4)
        self.assert_rows(data, 4)
        self.assert_cols(data, 2)

    def test_len_0x0 (self):
        data = self.data_from_str("")
        self.assertEqual(len(data), 0)
        self.assert_rows(data, 0)
        self.assert_cols(data, 0)

    def test_set_header (self):
        data = self.set_to_2x4_grid()

        self.assert_header(data, False)
        self.set_header(data)
        self.assert_header(data, True)

        self.assertEqual(len(data), 3)
        self.assert_rows(data, 3)
        self.assert_cols(data, 2)

    def test_values_3x3 (self):
        data = self.set_to_3x3_grid()

        self.assertEqual(data[0][0], "first")
        self.assertEqual(data[0][1], "second")
        self.assertEqual(data[0][2], "third")
        self.assertEqual(data[1][0], "matthew")
        self.assertEqual(data[1][1], "alexander")
        self.assertEqual(data[1][2], "lachance")
        self.assertEqual(data[2][0], "un (french for one)")
        self.assertEqual(data[2][1], "deux")
        self.assertEqual(data[2][2], "trois")

        for i in range(3):
            self.assertTrue(isinstance(data[i], tuple))

            with self.assertRaises(IndexError):
                should_not_exist = data[i][3]

        with self.assertRaises(IndexError):
            row_should_not_exist = data[3]

    def test_values_2x4 (self):
        data = self.set_to_2x4_grid()

        self.assertEqual(data[0][0], "first")
        self.assertEqual(data[0][1], "second")
        self.assertEqual(data[1][0], "matthew")
        self.assertEqual(data[1][1], "lachance")
        self.assertEqual(data[2][0], "flibbity flobbity")
        self.assertEqual(data[2][1], "floop")
        self.assertEqual(data[3][0], "un")
        self.assertEqual(data[3][1], "deux")

        for i in range(4):
            self.assertTrue(isinstance(data[i], tuple))

            with self.assertRaises(IndexError):
                should_not_exist = data[i][2]

        with self.assertRaises(IndexError):
            row_should_not_exist = data[4]

    def test_values_0x0 (self):
        data = self.data_from_str("")

        with self.assertRaises(IndexError):
            row_should_not_exist = data[0]

    def test_slice_accession (self):
        data = self.set_to_3x3_grid()
        self.assertTrue(isinstance(data[:], list))

        self.set_header(data)
        self.assertTrue(isinstance(data[:], list))

    def test_repr (self):
        data = self.set_to_3x3_grid()
        self.assertEqual(repr(data),
                "<{} header=False".format(self.repr_class_name)
                + " ('first', 'second', 'third'),"
                + " ('matthew', 'alexander', 'lachance'),"
                + " ('un (french for one)', 'deux', 'trois')>")

class TestTabularData (unittest.TestCase, BaseTabularDataTest):

    repr_class_name = "TabularData"

    def setUp (self):
        self.data_bytes = b""

    def init_tabular_data (self):
        """Return TabularData initted with our temporary file."""
        return TabularData(self.data_bytes)

    def write_file (self, content):
        """Shortcut to write bytes or a str to a file."""

        if isinstance(content, str):
            # If the content is a str, then I can write all like normal
            # at a high level.
            self.data_bytes = content.encode("utf-8")

        else:
            # Otherwise, I need to write bytes, and I should ensure that
            # I actually have bytes to write with.
            assert isinstance(content, bytes)
            self.data_bytes = content

    def test_is_sequence (self):
        self.assertTrue(issubclass(TabularData, Sequence))

class TestTabularDataFromFilePath (unittest.TestCase,
        BaseTabularDataTest):

    repr_class_name = "TabularDataFromFilePath"

    def setUp (self):
        # The TabularData class needs a path in the filesystem, so I
        # probably need to create a file.
        fd, self.path = mkstemp(dir="/tmp", suffix=".txt")

        # I don't want to deal with this low-level nonsense.
        os.close(fd)

    def tearDown (self):
        # Delete the temporary file I created.
        os.unlink(self.path)

    def init_tabular_data (self):
        """Return TabularData initted with our temporary file."""
        return TabularDataFromFilePath(self.path)

    def write_file (self, content):
        """Shortcut to write bytes or a str to a file."""

        if isinstance(content, str):
            # If the content is a str, then I can write all like normal
            # at a high level.
            mode = "w"

        else:
            # Otherwise, I need to write bytes, and I should ensure that
            # I actually have bytes to write with.
            assert isinstance(content, bytes)
            mode = "wb"

        with open(self.path, mode) as obj:
            # Write the content into the temporary file we've created at
            # setup.
            obj.write(content)

    def test_is_sequence (self):
        self.assertTrue(issubclass(TabularDataFromFilePath, TabularData))

class TestExceptions (unittest.TestCase):

    def test_input_file_error (self):
        self.assertTrue(issubclass(
            TabularData.InputFileError, BaseError))

    def test_cant_decode_encoding (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Can't figure out file encoding for /path/to/file"):
            raise TabularData.CantDecodeEncoding("/path/to/file")

    def test_invalid_controls (self):
        with self.assertRaisesRegex(TabularData.CantDecodeEncoding,
                r"Unexpected control character \(0x0d\) in filename"):
            raise TabularData.InvalidControls(13, "filename")

    def test_inconsistent_newlines (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Some combination of LF, CR, and CRLFs in filename"):
            raise TabularData.InconsistentNewlines("filename")

    def test_inconsistent_column_counts (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Expected each row to have the same column count " \
                "in filename"):
            raise TabularData.InconsistentColumnCounts("filename")
<commit_msg>Rename test<commit_after># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from collections.abc import Sequence
from tempfile import mkstemp
import os
import unittest

from ..tabular_data import TabularData, TabularDataFromFilePath
from ...exceptions import BaseError

class BaseTabularDataTest:

    def set_header (self, data, value = True):
        """Set the header to the given value (or to True)."""
        data.header = value

    def assert_header (self, data, value):
        """Get the data's header and assert its equality."""
        self.assertEqual(data.header, value)

    def assert_rows (self, data, value):
        """Get the data's row count and assert its equality."""
        self.assertEqual(data.rows, value)

    def assert_cols (self, data, value):
        """Get the data's column count and assert its equality."""
        self.assertEqual(data.cols, value)

    def data_from_str (self, content):
        """Shortcut for writing content to a file and opening it."""
        self.write_file(content)
        return self.init_tabular_data()

    def test_bad_bytes (self):
        for i in b"\x81\x8d\x8f\x90\x9d":
            self.write_file(i.to_bytes(1, "big"))
            with self.assertRaises(TabularData.CantDecodeEncoding):
                data = self.init_tabular_data()

    def test_bad_control_characters (self):
        ranges = (
            (0x00, 0x09), # skip HT, LF
            (0x0b, 0x0d), # skip CR
            (0x0e, 0x20), # skip [ -~] aka visible characters
            (0x7f, 0xa0), # not worrying about non-ASCII controls
        )

        for low, high in ranges:
            for i in range(low, high):
                self.write_file("something{}something\n".format(chr(i)))
                regex = "0x{:02x}".format(i)
                with self.assertRaisesRegex(TabularData.InvalidControls, regex):
                    data = self.init_tabular_data()

    def test_error_on_stupid_newlines (self):
        self.write_file("first line\nsecond line\r\nthird line\rfourth")

        with self.assertRaises(TabularData.InconsistentNewlines):
            data = self.init_tabular_data()

    def test_error_on_LF_and_CRLF (self):
        self.write_file("first line\nsecond line\r\nthird line")

        with self.assertRaises(TabularData.InconsistentNewlines):
            data = self.init_tabular_data()

    def test_error_on_inconsistent_columns (self):
        self.write_file("shipment\tvolume\n1234567\t8974326\t7843927")

        with self.assertRaises(TabularData.InconsistentColumnCounts):
            data = self.init_tabular_data()

    def set_to_3x3_grid (self):
        return self.data_from_str("\n".join((
                "first\tsecond\tthird",
                "matthew\talexander\tlachance",
                "un (french for one)\tdeux\ttrois")))

    def set_to_2x4_grid (self):
        return self.data_from_str("\n".join((
                "first\tsecond",
                "matthew\tlachance",
                "flibbity flobbity\tfloop",
                "un\tdeux")))

    def test_len_3x3 (self):
        data = self.set_to_3x3_grid()
        self.assertEqual(len(data), 3)
        self.assert_rows(data, 3)
        self.assert_cols(data, 3)

    def test_len_2x4 (self):
        data = self.set_to_2x4_grid()
        self.assertEqual(len(data), 4)
        self.assert_rows(data, 4)
        self.assert_cols(data, 2)

    def test_len_0x0 (self):
        data = self.data_from_str("")
        self.assertEqual(len(data), 0)
        self.assert_rows(data, 0)
        self.assert_cols(data, 0)

    def test_set_header (self):
        data = self.set_to_2x4_grid()

        self.assert_header(data, False)
        self.set_header(data)
        self.assert_header(data, True)

        self.assertEqual(len(data), 3)
        self.assert_rows(data, 3)
        self.assert_cols(data, 2)

    def test_values_3x3 (self):
        data = self.set_to_3x3_grid()

        self.assertEqual(data[0][0], "first")
        self.assertEqual(data[0][1], "second")
        self.assertEqual(data[0][2], "third")
        self.assertEqual(data[1][0], "matthew")
        self.assertEqual(data[1][1], "alexander")
        self.assertEqual(data[1][2], "lachance")
        self.assertEqual(data[2][0], "un (french for one)")
        self.assertEqual(data[2][1], "deux")
        self.assertEqual(data[2][2], "trois")

        for i in range(3):
            self.assertTrue(isinstance(data[i], tuple))

            with self.assertRaises(IndexError):
                should_not_exist = data[i][3]

        with self.assertRaises(IndexError):
            row_should_not_exist = data[3]

    def test_values_2x4 (self):
        data = self.set_to_2x4_grid()

        self.assertEqual(data[0][0], "first")
        self.assertEqual(data[0][1], "second")
        self.assertEqual(data[1][0], "matthew")
        self.assertEqual(data[1][1], "lachance")
        self.assertEqual(data[2][0], "flibbity flobbity")
        self.assertEqual(data[2][1], "floop")
        self.assertEqual(data[3][0], "un")
        self.assertEqual(data[3][1], "deux")

        for i in range(4):
            self.assertTrue(isinstance(data[i], tuple))

            with self.assertRaises(IndexError):
                should_not_exist = data[i][2]

        with self.assertRaises(IndexError):
            row_should_not_exist = data[4]

    def test_values_0x0 (self):
        data = self.data_from_str("")

        with self.assertRaises(IndexError):
            row_should_not_exist = data[0]

    def test_slice_accession (self):
        data = self.set_to_3x3_grid()
        self.assertTrue(isinstance(data[:], list))

        self.set_header(data)
        self.assertTrue(isinstance(data[:], list))

    def test_repr (self):
        data = self.set_to_3x3_grid()
        self.assertEqual(repr(data),
                "<{} header=False".format(self.repr_class_name)
                + " ('first', 'second', 'third'),"
                + " ('matthew', 'alexander', 'lachance'),"
                + " ('un (french for one)', 'deux', 'trois')>")

class TestTabularData (unittest.TestCase, BaseTabularDataTest):

    repr_class_name = "TabularData"

    def setUp (self):
        self.data_bytes = b""

    def init_tabular_data (self):
        """Return TabularData initted with our temporary file."""
        return TabularData(self.data_bytes)

    def write_file (self, content):
        """Shortcut to write bytes or a str to a file."""

        if isinstance(content, str):
            # If the content is a str, then I can write all like normal
            # at a high level.
            self.data_bytes = content.encode("utf-8")

        else:
            # Otherwise, I need to write bytes, and I should ensure that
            # I actually have bytes to write with.
            assert isinstance(content, bytes)
            self.data_bytes = content

    def test_is_sequence (self):
        self.assertTrue(issubclass(TabularData, Sequence))

class TestTabularDataFromFilePath (unittest.TestCase,
        BaseTabularDataTest):

    repr_class_name = "TabularDataFromFilePath"

    def setUp (self):
        # The TabularData class needs a path in the filesystem, so I
        # probably need to create a file.
        fd, self.path = mkstemp(dir="/tmp", suffix=".txt")

        # I don't want to deal with this low-level nonsense.
        os.close(fd)

    def tearDown (self):
        # Delete the temporary file I created.
        os.unlink(self.path)

    def init_tabular_data (self):
        """Return TabularData initted with our temporary file."""
        return TabularDataFromFilePath(self.path)

    def write_file (self, content):
        """Shortcut to write bytes or a str to a file."""

        if isinstance(content, str):
            # If the content is a str, then I can write all like normal
            # at a high level.
            mode = "w"

        else:
            # Otherwise, I need to write bytes, and I should ensure that
            # I actually have bytes to write with.
            assert isinstance(content, bytes)
            mode = "wb"

        with open(self.path, mode) as obj:
            # Write the content into the temporary file we've created at
            # setup.
            obj.write(content)

    def test_is_tabular_data (self):
        self.assertTrue(issubclass(TabularDataFromFilePath, TabularData))

class TestExceptions (unittest.TestCase):

    def test_input_file_error (self):
        self.assertTrue(issubclass(
            TabularData.InputFileError, BaseError))

    def test_cant_decode_encoding (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Can't figure out file encoding for /path/to/file"):
            raise TabularData.CantDecodeEncoding("/path/to/file")

    def test_invalid_controls (self):
        with self.assertRaisesRegex(TabularData.CantDecodeEncoding,
                r"Unexpected control character \(0x0d\) in filename"):
            raise TabularData.InvalidControls(13, "filename")

    def test_inconsistent_newlines (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Some combination of LF, CR, and CRLFs in filename"):
            raise TabularData.InconsistentNewlines("filename")

    def test_inconsistent_column_counts (self):
        with self.assertRaisesRegex(TabularData.InputFileError,
                "Expected each row to have the same column count " \
                "in filename"):
            raise TabularData.InconsistentColumnCounts("filename")
<|endoftext|>