<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import lxml.etree
import os
import requests

import glob
import utils

requests.packages.urllib3.disable_warnings()

def get_individual_atlantsolia_prices():
    relation = glob.ATLANTSOLIA_LOCATION_RELATION
    url = 'https://www.atlantsolia.is/stodvaverd/'
    res = requests.get(url, headers=utils.headers())
    html_text = res.content
    html = lxml.etree.fromstring(html_text, lxml.etree.HTMLParser())
    div_prices = html.find(('.//*[@id="content"]/div/div/div/div[2]/div/div/'
                            'table/tbody'))
    # Atlantsolia recently declared one of its stations to be without discount,
    # on their site that station is marked with a star (*) and the following
    # comment at the bottom:
    # "* Engir afslaettir gilda"
    discountless_stations = ('ao_008', )
    prices = {}
    for div_price in div_prices:
        key = relation[div_price[0][0].text]
        bensin95 = float(div_price[1][0].text.replace(',', '.'))
        diesel = float(div_price[2][0].text.replace(',', '.'))
        bensin95_discount = int(
            (bensin95 - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
        ) / 10.0
        diesel_discount = int(
            (diesel - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
        ) / 10.0
        if key in discountless_stations:
            bensin95_discount = None
            diesel_discount = None
        prices[key] = {
            'bensin95': bensin95,
            'diesel': diesel,
            'bensin95_discount': bensin95_discount,
            'diesel_discount': diesel_discount
        }
    return prices

def get_global_costco_prices():
    # Costco does not intend to show petrol price on their webpage according to
    # email reply I've received from them, so we have no choice but to hardcode
    # their prices if we wish to include their prices in Gasvaktin.
    # Instead of hardcoding prices in the repo itself like with Dælan last
    # summer I've created a personal Google Docs Spreadsheet to hold their
    # current petrol prices, Simplifies updating prices if away from
    # workstation.
    headers = {
        'Accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,'
            '*/*;q=0.8'
        ),
        'Accept-Encoding': 'gzip, deflate, sdch, br',
        'Accept-Language': 'en-US,en;q=0.8,is;q=0.6,da;q=0.4',
        'Cache-Control': 'no-cache',
        'Pragma': 'no-cache',
        'User-Agent': utils.random_ua()
    }
    url = (
        'https://docs.google.com/spreadsheets/d/'
        '18xuZbhfInW_6Loua3_4LE7KxbGPsh-_3IFfLpf3uwYE/edit'
    )
    res = requests.get(url, headers=headers)
    res.raise_for_status()
    # <shameless-incredibly-naive-html-parsing>
    bensin = None
    diesel = None
    html_text = res.content
    for line in html_text.split('\n'):
        if line.startswith(' Bensin,'):
            bensin = float(line[8:].replace(' ', ''))
        if line.startswith(' Diesel,'):
            diesel = float(line[8:].replace(' ', ''))
        if bensin is not None and diesel is not None:
            break
    if bensin is None or diesel is None:
        err_msg = 'Failed to read costco prices (%s, %s)\n%s ...' % (
            bensin,
            diesel,
            html_text[:10000]
        )
        raise Exception(err_msg)
    # </shameless-incredibly-naive-html-parsing>
    return {
        'bensin95': bensin,
        'diesel': diesel,
        # Costco has no discount program
        'bensin95_discount': None,
        'diesel_discount': None
    }

def get_global_n1_prices():
    url_eldsneyti = 'https://www.n1.is/thjonusta/eldsneyti'
    url_eldsneyti_api = 'https://www.n1.is/umbraco/api/fuel/GetSingleFuelPrice'
    headers = utils.headers()
    session = requests.Session()
    session.get(url_eldsneyti, headers=headers)
    post_data_bensin = 'fuelType=95+Oktan'
    post_data_diesel = 'fuelType=D%C3%ADsel'
    headers_eldsneyti_api = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.8,is;q=0.6',
        'Connection': 'keep-alive',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Host': 'www.n1.is',
        'Origin': 'https://www.n1.is',
        'Referer': 'https://www.n1.is/eldsneyti/',
        'User-Agent': headers['User-Agent'],
        'X-Requested-With': 'XMLHttpRequest'
    }
    headers_eldsneyti_api['Content-Length'] = str(len(post_data_bensin))
    res = session.post(url_eldsneyti_api, data=post_data_bensin,
                       headers=headers_eldsneyti_api)
    bensin95_discount_text = res.content
    headers_eldsneyti_api['Content-Length'] = str(len(post_data_diesel))
    res = session.post(url_eldsneyti_api, data=post_data_diesel,
                       headers=headers_eldsneyti_api)
    diesel_discount_text = res.content
    prices = {}
    prices['bensin95'] = float(
        bensin95_discount_text.replace('"', '').replace(',', '.'))
    prices['diesel'] = float(
        diesel_discount_text.replace('"', '').replace(',', '.'))
    prices['bensin95_discount'] = prices['bensin95'] - glob.N1_DISCOUNT
    prices['diesel_discount'] = prices['diesel'] - glob.N1_DISCOUNT
    return prices

def get_individual_daelan_prices():
    relation = glob.DAELAN_LOCATION_RELATION
    headers = utils.headers()
    session = requests.Session()
    session.get('http://daelan.is/', headers=headers)
    session.get('https://www.n1.is/', headers=headers)
    price_endpoint = 'https://www.n1.is/umbraco/api/Fuel/GetFuelPriceForDaelan'
    res = session.get(price_endpoint, headers=headers)
    res.raise_for_status()
    stations = res.json()
    prices = {}
    for station in stations:
        assert('location' in station)
        assert('gasPrice' in station)
        assert('diselPrice' in station)
        key = relation[station['location']]
        prices[key] = {
            'bensin95': float(station['gasPrice'].replace(',', '.')),
            'diesel': float(station['diselPrice'].replace(',', '.')),
            # Dælan has no discount program
            'bensin95_discount': None,
            'diesel_discount': None
        }
    return prices

def get_global_olis_prices():
    url = 'https://www.olis.is/solustadir/thjonustustodvar/eldsneytisverd/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
    diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
    bensin_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
    diesel_discount_text = html.find('.//*[@id="gas-price"]/span[5]').text
    return {
        'bensin95': float(bensin95_text.replace(',', '.')),
        'diesel': float(diesel_text.replace(',', '.')),
        'bensin95_discount': float(bensin_discount_text.replace(',', '.')),
        'diesel_discount': float(diesel_discount_text.replace(',', '.'))
    }

def get_individual_ob_prices():
    url = 'https://www.ob.is/eldsneytisverd/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
    diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
    bensin_discount_text = html.find('.//*[@id="gas-price"]/span[3]').text
    diesel_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
    bensin95 = float(bensin95_text.replace(',', '.'))
    diesel = float(diesel_text.replace(',', '.'))
    bensin95_discount = float(bensin_discount_text.replace(',', '.'))
    diesel_discount = float(diesel_discount_text.replace(',', '.'))
    prices = {}
    ob_stations = utils.load_json(
        os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            '../stations/ob.json'
        )
    )
    # 15 ISK discount for selected stations in September
    #
    # Occasionally, selected OB stations have had higher discount than usually
    # for a specified month, it seems they're going to do this every month from
    # now on (according to fb post in September).
    # facebook.com/ob.bensin/photos/a.208957995809394/1904154726289704/
    #
    # Stations with additional discount in September 2018:
    # * Baejarlind (ob_010)
    # * Fjardarkaup (ob_012)
    # * Njardvik (ob_024)
    # * Starengi (ob_029)
    # * Borgarnes (ob_009)
    additional_discount = 15
    additional_discount_stations = (
        'ob_010', 'ob_012', 'ob_024', 'ob_029', 'ob_009',
    )
    now = datetime.datetime.now()
    end = datetime.datetime.strptime('2018-09-30T23:59', '%Y-%m-%dT%H:%M')
    for key in ob_stations:
        bensin95_discount = bensin95_discount
        diesel_discount = diesel_discount
        if now < end and key in additional_discount_stations:
            bensin95_discount = bensin95 - additional_discount
            diesel_discount = diesel - additional_discount
        prices[key] = {
            'bensin95': bensin95,
            'diesel': diesel,
            'bensin95_discount': bensin95_discount,
            'diesel_discount': diesel_discount
        }
    return prices

def get_individual_orkan_prices():
    # Read prices for Orkan and Orkan X stations because they're both on the
    # same webpage.
    url = 'https://www.orkan.is/orkan/orkustodvar/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    div_element = html.find('.//div[@class="accordion__container"]')
    territories = div_element.findall('.//div[@class="accordion__child"]')
    prices = {}
    key = None
    for territory in territories:
        for station in territory:
            station_name = station[0][0].text
            bensin95 = float(station[1].text.replace(',', '.'))
            diesel = float(station[2].text.replace(',', '.'))
            if station_name in glob.ORKAN_LOCATION_RELATION:
                key = glob.ORKAN_LOCATION_RELATION[station_name]
            elif station_name in glob.ORKAN_X_LOCATION_RELATION:
                key = glob.ORKAN_X_LOCATION_RELATION[station_name]
            else:
                continue
            # Orkan X stations have key starting with "ox", while ordinary
            # Orkan statins have keys starting with "or"
            bensin95_discount = None
            diesel_discount = None
            if key.startswith('or'):
                # Orkan has a 3-step discount system controlled by your
                # spendings on gas from them the month before
                # See more info here: https://www.orkan.is/Afslattarthrep
                # For consistency we just use the minimum default discount
                bensin95_discount = bensin95 - glob.ORKAN_MINIMUM_DISCOUNT
                diesel_discount = diesel - glob.ORKAN_MINIMUM_DISCOUNT
            prices[key] = {
                'bensin95': bensin95,
                'diesel': diesel,
                'bensin95_discount': bensin95_discount,
                'diesel_discount': diesel_discount
            }
    # TEMPORARY_FIX while three former Skeljungur stations are missing from the
    # list of stations on Orkan webpage
    for key in ['or_053', 'or_054', 'or_055']:
        if key not in prices:
            prices[key] = prices['or_000'].copy()
            if key in ['or_054', 'or_055']:
                # diesel currently 1 ISK more expensive on these two stations
                prices[key]['diesel'] += 1
                prices[key]['diesel_discount'] += 1
    # /TEMPORARY_FIX
    return prices

if __name__ == '__main__':
    print 'Testing scrapers\n'
    print 'Atlantsolía'
    print get_individual_atlantsolia_prices()
    print 'Costco'
    print get_global_costco_prices()
    print 'Dælan'
    print get_individual_daelan_prices()
    print 'N1'
    print get_global_n1_prices()
    print 'Olís'
    print get_global_olis_prices()
    print 'ÓB'
    print get_individual_ob_prices()
    print 'Orkan (including Orkan X)'
    print get_individual_orkan_prices()
<commit_msg>Costco price read fix something google docs html related changed, updating shameless-incredibly-naive-html-parsing to compensate<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import lxml.etree
import os
import requests

import glob
import utils

requests.packages.urllib3.disable_warnings()

def get_individual_atlantsolia_prices():
    relation = glob.ATLANTSOLIA_LOCATION_RELATION
    url = 'https://www.atlantsolia.is/stodvaverd/'
    res = requests.get(url, headers=utils.headers())
    html_text = res.content
    html = lxml.etree.fromstring(html_text, lxml.etree.HTMLParser())
    div_prices = html.find(('.//*[@id="content"]/div/div/div/div[2]/div/div/'
                            'table/tbody'))
    # Atlantsolia recently declared one of its stations to be without discount,
    # on their site that station is marked with a star (*) and the following
    # comment at the bottom:
    # "* Engir afslaettir gilda"
    discountless_stations = ('ao_008', )
    prices = {}
    for div_price in div_prices:
        key = relation[div_price[0][0].text]
        bensin95 = float(div_price[1][0].text.replace(',', '.'))
        diesel = float(div_price[2][0].text.replace(',', '.'))
        bensin95_discount = int(
            (bensin95 - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
        ) / 10.0
        diesel_discount = int(
            (diesel - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
        ) / 10.0
        if key in discountless_stations:
            bensin95_discount = None
            diesel_discount = None
        prices[key] = {
            'bensin95': bensin95,
            'diesel': diesel,
            'bensin95_discount': bensin95_discount,
            'diesel_discount': diesel_discount
        }
    return prices

def get_global_costco_prices():
    # Costco does not intend to show petrol price on their webpage according to
    # email reply I've received from them, so we have no choice but to hardcode
    # their prices if we wish to include their prices in Gasvaktin.
    # Instead of hardcoding prices in the repo itself like with Dælan last
    # summer I've created a personal Google Docs Spreadsheet to hold their
    # current petrol prices, Simplifies updating prices if away from
    # workstation.
    headers = {
        'Accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,'
            '*/*;q=0.8'
        ),
        'Accept-Encoding': 'gzip, deflate, sdch, br',
        'Accept-Language': 'en-US,en;q=0.8,is;q=0.6,da;q=0.4',
        'Cache-Control': 'no-cache',
        'Pragma': 'no-cache',
        'User-Agent': utils.random_ua()
    }
    url = (
        'https://docs.google.com/spreadsheets/d/'
        '18xuZbhfInW_6Loua3_4LE7KxbGPsh-_3IFfLpf3uwYE/edit'
    )
    res = requests.get(url, headers=headers)
    res.raise_for_status()
    # <shameless-incredibly-naive-html-parsing>
    bensin = None
    diesel = None
    html_text = res.content
    for line in html_text.split('\n'):
        if line.lstrip().startswith('Bensin,'):
            bensin = float(line.lstrip()[7:].replace(' ', ''))
        if line.lstrip().startswith('Diesel,'):
            diesel = float(line.lstrip()[7:].replace(' ', ''))
        if bensin is not None and diesel is not None:
            break
    if bensin is None or diesel is None:
        err_msg = 'Failed to read costco prices (%s, %s)\n%s ...' % (
            bensin,
            diesel,
            html_text[:10000]
        )
        raise Exception(err_msg)
    # </shameless-incredibly-naive-html-parsing>
    return {
        'bensin95': bensin,
        'diesel': diesel,
        # Costco has no discount program
        'bensin95_discount': None,
        'diesel_discount': None
    }

def get_global_n1_prices():
    url_eldsneyti = 'https://www.n1.is/thjonusta/eldsneyti'
    url_eldsneyti_api = 'https://www.n1.is/umbraco/api/fuel/GetSingleFuelPrice'
    headers = utils.headers()
    session = requests.Session()
    session.get(url_eldsneyti, headers=headers)
    post_data_bensin = 'fuelType=95+Oktan'
    post_data_diesel = 'fuelType=D%C3%ADsel'
    headers_eldsneyti_api = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.8,is;q=0.6',
        'Connection': 'keep-alive',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Host': 'www.n1.is',
        'Origin': 'https://www.n1.is',
        'Referer': 'https://www.n1.is/eldsneyti/',
        'User-Agent': headers['User-Agent'],
        'X-Requested-With': 'XMLHttpRequest'
    }
    headers_eldsneyti_api['Content-Length'] = str(len(post_data_bensin))
    res = session.post(url_eldsneyti_api, data=post_data_bensin,
                       headers=headers_eldsneyti_api)
    bensin95_discount_text = res.content
    headers_eldsneyti_api['Content-Length'] = str(len(post_data_diesel))
    res = session.post(url_eldsneyti_api, data=post_data_diesel,
                       headers=headers_eldsneyti_api)
    diesel_discount_text = res.content
    prices = {}
    prices['bensin95'] = float(
        bensin95_discount_text.replace('"', '').replace(',', '.'))
    prices['diesel'] = float(
        diesel_discount_text.replace('"', '').replace(',', '.'))
    prices['bensin95_discount'] = prices['bensin95'] - glob.N1_DISCOUNT
    prices['diesel_discount'] = prices['diesel'] - glob.N1_DISCOUNT
    return prices

def get_individual_daelan_prices():
    relation = glob.DAELAN_LOCATION_RELATION
    headers = utils.headers()
    session = requests.Session()
    session.get('http://daelan.is/', headers=headers)
    session.get('https://www.n1.is/', headers=headers)
    price_endpoint = 'https://www.n1.is/umbraco/api/Fuel/GetFuelPriceForDaelan'
    res = session.get(price_endpoint, headers=headers)
    res.raise_for_status()
    stations = res.json()
    prices = {}
    for station in stations:
        assert('location' in station)
        assert('gasPrice' in station)
        assert('diselPrice' in station)
        key = relation[station['location']]
        prices[key] = {
            'bensin95': float(station['gasPrice'].replace(',', '.')),
            'diesel': float(station['diselPrice'].replace(',', '.')),
            # Dælan has no discount program
            'bensin95_discount': None,
            'diesel_discount': None
        }
    return prices

def get_global_olis_prices():
    url = 'https://www.olis.is/solustadir/thjonustustodvar/eldsneytisverd/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
    diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
    bensin_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
    diesel_discount_text = html.find('.//*[@id="gas-price"]/span[5]').text
    return {
        'bensin95': float(bensin95_text.replace(',', '.')),
        'diesel': float(diesel_text.replace(',', '.')),
        'bensin95_discount': float(bensin_discount_text.replace(',', '.')),
        'diesel_discount': float(diesel_discount_text.replace(',', '.'))
    }

def get_individual_ob_prices():
    url = 'https://www.ob.is/eldsneytisverd/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
    diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
    bensin_discount_text = html.find('.//*[@id="gas-price"]/span[3]').text
    diesel_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
    bensin95 = float(bensin95_text.replace(',', '.'))
    diesel = float(diesel_text.replace(',', '.'))
    bensin95_discount = float(bensin_discount_text.replace(',', '.'))
    diesel_discount = float(diesel_discount_text.replace(',', '.'))
    prices = {}
    ob_stations = utils.load_json(
        os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            '../stations/ob.json'
        )
    )
    # 15 ISK discount for selected stations in September
    #
    # Occasionally, selected OB stations have had higher discount than usually
    # for a specified month, it seems they're going to do this every month from
    # now on (according to fb post in September).
    # facebook.com/ob.bensin/photos/a.208957995809394/1904154726289704/
    #
    # Stations with additional discount in September 2018:
    # * Baejarlind (ob_010)
    # * Fjardarkaup (ob_012)
    # * Njardvik (ob_024)
    # * Starengi (ob_029)
    # * Borgarnes (ob_009)
    additional_discount = 15
    additional_discount_stations = (
        'ob_010', 'ob_012', 'ob_024', 'ob_029', 'ob_009',
    )
    now = datetime.datetime.now()
    end = datetime.datetime.strptime('2018-09-30T23:59', '%Y-%m-%dT%H:%M')
    for key in ob_stations:
        bensin95_discount = bensin95_discount
        diesel_discount = diesel_discount
        if now < end and key in additional_discount_stations:
            bensin95_discount = bensin95 - additional_discount
            diesel_discount = diesel - additional_discount
        prices[key] = {
            'bensin95': bensin95,
            'diesel': diesel,
            'bensin95_discount': bensin95_discount,
            'diesel_discount': diesel_discount
        }
    return prices

def get_individual_orkan_prices():
    # Read prices for Orkan and Orkan X stations because they're both on the
    # same webpage.
    url = 'https://www.orkan.is/orkan/orkustodvar/'
    res = requests.get(url, headers=utils.headers(), verify=False)
    html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
    div_element = html.find('.//div[@class="accordion__container"]')
    territories = div_element.findall('.//div[@class="accordion__child"]')
    prices = {}
    key = None
    for territory in territories:
        for station in territory:
            station_name = station[0][0].text
            bensin95 = float(station[1].text.replace(',', '.'))
            diesel = float(station[2].text.replace(',', '.'))
            if station_name in glob.ORKAN_LOCATION_RELATION:
                key = glob.ORKAN_LOCATION_RELATION[station_name]
            elif station_name in glob.ORKAN_X_LOCATION_RELATION:
                key = glob.ORKAN_X_LOCATION_RELATION[station_name]
            else:
                continue
            # Orkan X stations have key starting with "ox", while ordinary
            # Orkan statins have keys starting with "or"
            bensin95_discount = None
            diesel_discount = None
            if key.startswith('or'):
                # Orkan has a 3-step discount system controlled by your
                # spendings on gas from them the month before
                # See more info here: https://www.orkan.is/Afslattarthrep
                # For consistency we just use the minimum default discount
                bensin95_discount = bensin95 - glob.ORKAN_MINIMUM_DISCOUNT
                diesel_discount = diesel - glob.ORKAN_MINIMUM_DISCOUNT
            prices[key] = {
                'bensin95': bensin95,
                'diesel': diesel,
                'bensin95_discount': bensin95_discount,
                'diesel_discount': diesel_discount
            }
    # TEMPORARY_FIX while three former Skeljungur stations are missing from the
    # list of stations on Orkan webpage
    for key in ['or_053', 'or_054', 'or_055']:
        if key not in prices:
            prices[key] = prices['or_000'].copy()
            if key in ['or_054', 'or_055']:
                # diesel currently 1 ISK more expensive on these two stations
                prices[key]['diesel'] += 1
                prices[key]['diesel_discount'] += 1
    # /TEMPORARY_FIX
    return prices

if __name__ == '__main__':
    print 'Testing scrapers\n'
    print 'Atlantsolía'
    print get_individual_atlantsolia_prices()
    print 'Costco'
    print get_global_costco_prices()
    print 'Dælan'
    print get_individual_daelan_prices()
    print 'N1'
    print get_global_n1_prices()
    print 'Olís'
    print get_global_olis_prices()
    print 'ÓB'
    print get_individual_ob_prices()
    print 'Orkan (including Orkan X)'
    print get_individual_orkan_prices()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Unit tests for cartoframes.contrib.vector"""
import unittest
import os
import sys
import json
import random
import warnings
import requests

import cartoframes
from cartoframes.contrib import vector
from carto.exceptions import CartoException
from carto.auth import APIKeyAuthClient
from carto.sql import SQLClient
from pyrestcli.exceptions import NotFoundException
import pandas as pd
try:
    import geopandas as gpd
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False
from IPython.display import HTML

WILL_SKIP = False
warnings.filterwarnings("ignore")

class _UserUrlLoader:
    def user_url(self):
        user_url = None
        if (os.environ.get('USERURL') is None):
            try:
                creds = json.loads(open('test/secret.json').read())
                user_url = creds['USERURL']
            except:
                warnings.warn('secret.json not found')

        if user_url in (None, ''):
            user_url = 'https://{username}.carto.com/'

        return user_url

class TestContribVector(unittest.TestCase, _UserUrlLoader):
    """Tests for cartoframes.CartoContext"""
    def setUp(self):
        if (os.environ.get('APIKEY') is None or
                os.environ.get('USERNAME') is None):
            try:
                creds = json.loads(open('test/secret.json').read())
                self.apikey = creds['APIKEY']
                self.username = creds['USERNAME']
            except:  # noqa: E722
                warnings.warn("Skipping CartoContext tests. To test it, "
                              "create a `secret.json` file in test/ by "
                              "renaming `secret.json.sample` to `secret.json` "
                              "and updating the credentials to match your "
                              "environment.")
                self.apikey = None
                self.username = None
        else:
            self.apikey = os.environ['APIKEY']
            self.username = os.environ['USERNAME']

        self.user_url = self.user_url()

        if self.username and self.apikey:
            self.baseurl = self.user_url.format(
                    username=self.username)
            self.auth_client = APIKeyAuthClient(base_url=self.baseurl,
                                                api_key=self.apikey)
            self.sql_client = SQLClient(self.auth_client)

        # sets client to be ci
        if not cartoframes.context.DEFAULT_SQL_ARGS['client']\
                .endswith('_dev_ci'):
            cartoframes.context.DEFAULT_SQL_ARGS['client'] += '_dev_ci'
        # sets skip value
        WILL_SKIP = self.apikey is None or self.username is None  # noqa: F841

        self.points = 'tweets_obama'
        self.polys = 'nat'
        self.local = 'cb_2013_us_csa_500k'

    def test_vector_multilayer(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        layers = [
            vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                time='torque($cartodb_id, 10)', strokeWidth=2)
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

    @unittest.skipIf(not HAS_GEOPANDAS, 'GeoPandas not installed')
    def test_vector_local(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        df = gpd.GeoDataFrame(
            cc.query('''
                select the_geom, the_geom_webmercator, cartodb_id
                from {}
            '''.format(self.local),
            decode_geom=True)
        )
        layers = [
            vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                time='torque($cartodb_id, 10)', strokeWidth=2),
            vector.LocalLayer(df)
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

    @unittest.skipIf(HAS_GEOPANDAS, 'GeoPandas installed, so not needed')
    def test_vector_local(self):
        """contrib.vector local with"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        with self.assertRaises(ValueError):
            df = cc.query('''
                    select the_geom, the_geom_webmercator, cartodb_id
                    from {}
                '''.format(self.local),
                decode_geom=True)
            layers = [
                vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
                vector.QueryLayer(
                    'SELECT * FROM {}'.format(self.polys),
                    time='torque($cartodb_id, 10)', strokeWidth=2),
                vector.LocalLayer(df)
            ]
            self.assertIsInstance(vector.vmap(layers, cc), HTML)

    def test_vector_interactivity(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        layers = [
            vector.Layer(self.points, interactivity='body'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                interactivity=['name', 'state_name', ]),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                interactivity={
                    'cols': ['name', 'state_name', ],
                    'header': '
NAT
',
                    'event': 'click'
                })
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

        # invalid entry for interactivity
        with self.assertRaises(ValueError):
            vector.vmap([vector.Layer(self.points, interactivity=10), ])
<commit_msg>adds tests for new features<commit_after># -*- coding: utf-8 -*-
"""Unit tests for cartoframes.contrib.vector"""
import unittest
import os
import sys
import json
import random
import warnings
import requests

import cartoframes
from cartoframes.contrib import vector
from carto.exceptions import CartoException
from carto.auth import APIKeyAuthClient
from carto.sql import SQLClient
from pyrestcli.exceptions import NotFoundException
import pandas as pd
try:
    import geopandas as gpd
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False
from IPython.display import HTML

WILL_SKIP = False
warnings.filterwarnings("ignore")

class _UserUrlLoader:
    def user_url(self):
        user_url = None
        if (os.environ.get('USERURL') is None):
            try:
                creds = json.loads(open('test/secret.json').read())
                user_url = creds['USERURL']
            except:
                warnings.warn('secret.json not found')

        if user_url in (None, ''):
            user_url = 'https://{username}.carto.com/'

        return user_url

class TestContribVector(unittest.TestCase, _UserUrlLoader):
    """Tests for cartoframes.CartoContext"""
    def setUp(self):
        if (os.environ.get('APIKEY') is None or
                os.environ.get('USERNAME') is None):
            try:
                creds = json.loads(open('test/secret.json').read())
                self.apikey = creds['APIKEY']
                self.username = creds['USERNAME']
            except:  # noqa: E722
                warnings.warn("Skipping CartoContext tests. To test it, "
                              "create a `secret.json` file in test/ by "
                              "renaming `secret.json.sample` to `secret.json` "
                              "and updating the credentials to match your "
                              "environment.")
                self.apikey = None
                self.username = None
        else:
            self.apikey = os.environ['APIKEY']
            self.username = os.environ['USERNAME']

        self.user_url = self.user_url()
        self.baseurl = self.user_url

        if self.username and self.apikey:
            self.baseurl = self.user_url.format(
                    username=self.username)
            self.auth_client = APIKeyAuthClient(base_url=self.baseurl,
                                                api_key=self.apikey)
            self.sql_client = SQLClient(self.auth_client)

        # sets client to be ci
        if not cartoframes.context.DEFAULT_SQL_ARGS['client']\
                .endswith('_dev_ci'):
            cartoframes.context.DEFAULT_SQL_ARGS['client'] += '_dev_ci'
        # sets skip value
        WILL_SKIP = self.apikey is None or self.username is None  # noqa: F841

        self.points = 'tweets_obama'
        self.polys = 'nat'
        self.local = 'cb_2013_us_csa_500k'

    def test_vector_multilayer(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.user_url,
                                      api_key=self.apikey)
        layers = [
            vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                time='torque($cartodb_id, 10)', strokeWidth=2)
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

    @unittest.skipIf(not HAS_GEOPANDAS, 'GeoPandas not installed')
    def test_vector_local(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        df = gpd.GeoDataFrame(
            cc.query('''
                select the_geom, the_geom_webmercator, cartodb_id
                from {}
            '''.format(self.local),
            decode_geom=True)
        )
        layers = [
            vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                time='torque($cartodb_id, 10)', strokeWidth=2),
            vector.LocalLayer(df)
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

    @unittest.skipIf(HAS_GEOPANDAS, 'GeoPandas installed, so not needed')
    def test_vector_local(self):
        """contrib.vector local with"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        with self.assertRaises(ValueError):
            df = cc.query('''
                    select the_geom, the_geom_webmercator, cartodb_id
                    from {}
                '''.format(self.local),
                decode_geom=True)
            layers = [
                vector.Layer(self.points, color='red', size=10, strokeColor='blue'),
                vector.QueryLayer(
                    'SELECT * FROM {}'.format(self.polys),
                    time='torque($cartodb_id, 10)', strokeWidth=2),
                vector.LocalLayer(df)
            ]
            self.assertIsInstance(vector.vmap(layers, cc), HTML)

    def test_vector_interactivity(self):
        """contrib.vector"""
        cc = cartoframes.CartoContext(base_url=self.baseurl,
                                      api_key=self.apikey)
        layers = [
            vector.Layer(self.points, interactivity='body'),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                interactivity=['name', 'state_name', ]),
            vector.QueryLayer(
                'SELECT * FROM {}'.format(self.polys),
                interactivity={
                    'cols': ['name', 'state_name', ],
                    'header': '
NAT
',
                    'event': 'click'
                })
        ]
        self.assertIsInstance(vector.vmap(layers, cc), HTML)

        # invalid entry for interactivity
        with self.assertRaises(ValueError):
            vector.vmap([vector.Layer(self.points, interactivity=10), ])

    def test_vector_basemaps(self):
        """contrib.vector.BaseMaps"""
        self.assertEquals(vector.BaseMaps.positron, 'Positron')
        self.assertEquals(vector.BaseMaps.darkmatter, 'DarkMatter')
        self.assertEquals(vector.BaseMaps.voyager, 'Voyager')
        with self.assertRaises(AttributeError):
            b = vector.BaseMaps.doesntexist

    def test_vector_vmap_basemap(self):
        """contrib.vector.vmap with basemap flag"""
        cc = cartoframes.CartoContext(
            base_url=self.user_url,
            api_key=self.apikey
        )
        with self.assertRaises(ValueError, msg='style key not in basemap'):
            vector.vmap(
                [vector.Layer(self.points), ],
                context=cc,
                basemap={'tiles': 'abc123'}
            )

        vector.vmap(
            [vector.Layer(self.points), ],
            context=cc,
            basemap={'style': 'mapbox://styles/mapbox/streets-v9'}
        )
<|endoftext|>


<commit_before>import sys
import os.path

try:
    from . import GrammarConverter
except (SystemError, ImportError):
    import GrammarConverter

class Node:
    """
    Used for storing information about a non-terminal symbol. A node can have a maximum of two
    children because of the CNF of the grammar.
    It is possible though that there are multiple parses of a sentence. In this case information
    about an alternative child is stored in self.child1 or self.child2 (the parser will decide
    where according to the ambiguous rule).
    Either child1 is a terminal symbol passed as string, or both children are Nodes.
    """
    def __init__(self, symbol, child1, child2 = None):
        self.symbol = symbol
        self.child1 = child1
        self.child2 = child2

    def __repr__(self):
        """
        :return: the string representation of a Node object.
        """
        return self.symbol

class Parser:
    """
    A CYK parser which is able to parse any grammar in CNF. The grammar can be read from a file or
    passed as a string. It either returns a string representation of the parse tree(s) or prints it
    to standard out.
    """
    def __init__(self, grammar, sentence):
        """
        Creates a new parser object which will read in the grammar and transform it into CNF and
        then parse the given sentence with that grammar.
        :param grammar: the file path to the grammar/the string repr. of the grammar to read in
        :param sentence: the file path to the sentence/the string repr. of the sentence to read in
        """
        self.parse_table = None
        self.prods = {}
        self.grammar = None
        if os.path.isfile(grammar):
            self.grammar_from_file(grammar)
        else:
            self.grammar_from_string(grammar)
        self.__call__(sentence)

    def __call__(self, sentence, parse = False):
        """
        Parse the given sentence (string or file) with the earlier given grammar.
        :param sentence: the sentence to parse with self.grammar
        """
        if os.path.isfile(sentence):
            with open(sentence) as inp:
                self.input = inp.readline().split()
                if parse:
                    self.parse()
        else:
            self.input = sentence.split()

    def grammar_from_file(self, grammar):
        """
        Reads in a CFG from a given file, converts it to CNF and stores it in self.grammar.
        :param grammar: the file in which the grammar is stored.
        """
        self.grammar = GrammarConverter.convert_grammar(GrammarConverter.read_grammar(grammar))

    def grammar_from_string(self, grammar):
        """
        Reads in a CFG from a string, converts it to CNF and stores it in self.grammar.
        :param grammar: the CFG in string representation.
        :return:
        """
        self.grammar = GrammarConverter.convert_grammar(
            [x.replace("->", "").split() for x in grammar.split("\n")])

    def parse(self):
        """
        Does the actual parsing according to the CYK algorithm. The parse table is stored in
        self.parse_table.
        """
        length = len(self.input)
        # self.parse_table[y][x] is the list of nodes in the x+1 cell of y+1 row in the table.
        # That cell covers the word below it and y more words after.
        self.parse_table = [[[] for x in range(length - y)] for y in range(length)]

        for i, word in enumerate(self.input):
            # Find out which non terminals can generate the terminals in the input string
            # and put them into the parse table. One terminal could be generated by multiple
            # non terminals, therefore the parse table will contain a list of non terminals.
            for rule in self.grammar:
                if f"'{word}'" == rule[1]:
                    self.parse_table[0][i].append(Node(rule[0], word))
        for words_to_consider in range(2, length + 1):
            for starting_cell in range(0, length - words_to_consider + 1):
                for left_size in range(1, words_to_consider):
                    right_size = words_to_consider - left_size

                    left_cell = self.parse_table[left_size - 1][starting_cell]
                    right_cell = self.parse_table[right_size - 1][starting_cell + left_size]

                    for rule in self.grammar:
                        left_nodes = [n for n in left_cell if n.symbol == rule[1]]
                        if left_nodes:
                            right_nodes = [n for n in right_cell if n.symbol == rule[2]]
                            self.parse_table[words_to_consider - 1][starting_cell].extend(
                                [Node(rule[0], left, right) for left in left_nodes for right in right_nodes]
                            )

    def print_tree(self, output = True):
        """
        Print the parse tree starting with the start symbol. Alternatively it returns the string
        representation of the tree(s) instead of printing it.
        """
        start_symbol = self.grammar[0][0]

        final_nodes = [n for n in self.parse_table[-1][0] if n.symbol == start_symbol]
        if final_nodes:
            if output:
                print("The given sentence is contained in the language produced by the given "
                      "grammar!")
                print("\nPossible parse(s):")
            trees = [generate_trees(node) for node in final_nodes]
            if output:
                for tree in trees:
                    print(tree)
            else:
                return trees
        else:
            print("The given sentence is not contained in the language produced by the given "
                  "grammar!")

def generate_trees(node):
    """
    Generates a list the string representation of the parse tree(s).
    :param nodes: the root node.
    :return: the parse tree in string form.
    """
    if node.child2 == None:
        return f"[{node.symbol} '{node.child1}']"
    else:
        return f"[{node.symbol} {generate_trees(node.child1)} {generate_trees(node.child2)}]"

if __name__ == '__main__':
    ARGUMENTS = sys.argv
    if len(ARGUMENTS) > 3 or len(ARGUMENTS) == 2:
        print("Usage: python3 Parser.py <grammar.file> <sentence.file>\n"
              "or: python3 Parser.py <grammar as string> <sentence as string>")
    elif len(ARGUMENTS) == 3:
        GRAMMAR = "as path for the grammar" if os.path.isfile(ARGUMENTS[1]) else "as grammar"
        SENTENCE = "as path for the sentence" if os.path.isfile(ARGUMENTS[2]) else "as sentence"
        print(f"Using {ARGUMENTS[1]} {GRAMMAR} and {ARGUMENTS[2]} {SENTENCE}.")
        PARSER = Parser(ARGUMENTS[1], ARGUMENTS[2])
        PARSER.parse()
        PARSER.print_tree()
<commit_msg>fix naming and comments<commit_after>import sys
import os.path

try:
    from . import GrammarConverter
except (SystemError, ImportError):
    import GrammarConverter

class Node:
    """
    Used for storing information about a non-terminal symbol. A node can have a maximum of two
    children because of the CNF of the grammar.
    It is possible though that there are multiple parses of a sentence. In this case information
    about an alternative child is stored in self.child1 or self.child2 (the parser will decide
    where according to the ambiguous rule).
    Either child1 is a terminal symbol passed as string, or both children are Nodes.
    """
    def __init__(self, symbol, child1, child2 = None):
        self.symbol = symbol
        self.child1 = child1
        self.child2 = child2

    def __repr__(self):
        """
        :return: the string representation of a Node object.
        """
        return self.symbol

class Parser:
    """
    A CYK parser which is able to parse any grammar in CNF. The grammar can be read from a file or
    passed as a string. It either returns a string representation of the parse tree(s) or prints it
    to standard out.
    """
    def __init__(self, grammar, sentence):
        """
        Creates a new parser object which will read in the grammar and transform it into CNF and
        then parse the given sentence with that grammar.
        :param grammar: the file path to the grammar/the string repr. of the grammar to read in
        :param sentence: the file path to the sentence/the string repr. of the sentence to read in
        """
        self.parse_table = None
        self.prods = {}
        self.grammar = None
        if os.path.isfile(grammar):
            self.grammar_from_file(grammar)
        else:
            self.grammar_from_string(grammar)
        self.__call__(sentence)

    def __call__(self, sentence, parse = False):
        """
        Parse the given sentence (string or file) with the earlier given grammar.
        :param sentence: the sentence to parse with self.grammar
        """
        if os.path.isfile(sentence):
            with open(sentence) as inp:
                self.input = inp.readline().split()
                if parse:
                    self.parse()
        else:
            self.input = sentence.split()

    def grammar_from_file(self, grammar):
        """
        Reads in a CFG from a given file, converts it to CNF and stores it in self.grammar.
        :param grammar: the file in which the grammar is stored.
        """
        self.grammar = GrammarConverter.convert_grammar(GrammarConverter.read_grammar(grammar))

    def grammar_from_string(self, grammar):
        """
        Reads in a CFG from a string, converts it to CNF and stores it in self.grammar.
        :param grammar: the CFG in string representation.
        :return:
        """
        self.grammar = GrammarConverter.convert_grammar(
            [x.replace("->", "").split() for x in grammar.split("\n")])

    def parse(self):
        """
        Does the actual parsing according to the CYK algorithm. The parse table is stored in
        self.parse_table.
        """
        length = len(self.input)
        # self.parse_table[y][x] is the list of nodes in the x+1 cell of y+1 row in the table.
        # That cell covers the word below it and y more words after.
        self.parse_table = [[[] for x in range(length - y)] for y in range(length)]

        for i, word in enumerate(self.input):
            # Find out which non terminals can generate the terminals in the input string
            # and put them into the parse table. One terminal could be generated by multiple
            # non terminals, therefore the parse table will contain a list of non terminals.
            for rule in self.grammar:
                if f"'{word}'" == rule[1]:
                    self.parse_table[0][i].append(Node(rule[0], word))
        for words_to_consider in range(2, length + 1):
            for starting_cell in range(0, length - words_to_consider + 1):
                for left_size in range(1, words_to_consider):
                    right_size = words_to_consider - left_size

                    left_cell = self.parse_table[left_size - 1][starting_cell]
                    right_cell = self.parse_table[right_size - 1][starting_cell + left_size]

                    for rule in self.grammar:
                        left_nodes = [n for n in left_cell if n.symbol == rule[1]]
                        if left_nodes:
                            right_nodes = [n for n in right_cell if n.symbol == rule[2]]
                            self.parse_table[words_to_consider - 1][starting_cell].extend(
                                [Node(rule[0], left, right) for left in left_nodes for right in right_nodes]
                            )

    def print_tree(self, output = True):
        """
        Print the parse tree starting with the start symbol. Alternatively it returns the string
        representation of the tree(s) instead of printing it.
        """
        start_symbol = self.grammar[0][0]

        final_nodes = [n for n in self.parse_table[-1][0] if n.symbol == start_symbol]
        if final_nodes:
            if output:
                print("The given sentence is contained in the language produced by the given "
                      "grammar!")
                print("\nPossible parse(s):")
            trees = [generate_tree(node) for node in final_nodes]
            if output:
                for tree in trees:
                    print(tree)
            else:
                return trees
        else:
            print("The given sentence is not contained in the language produced by the given "
                  "grammar!")

def generate_tree(node):
    """
    Generates the string representation of the parse tree.
    :param nodes: the root node.
    :return: the parse tree in string form.
    """
    if node.child2 == None:
        return f"[{node.symbol} '{node.child1}']"
    else:
        return f"[{node.symbol} {generate_tree(node.child1)} {generate_tree(node.child2)}]"

if __name__ == '__main__':
    ARGUMENTS = sys.argv
    if len(ARGUMENTS) > 3 or len(ARGUMENTS) == 2:
        print("Usage: python3 Parser.py <grammar.file> <sentence.file>\n"
              "or: python3 Parser.py <grammar as string> <sentence as string>")
    elif len(ARGUMENTS) == 3:
        GRAMMAR = "as path for the grammar" if os.path.isfile(ARGUMENTS[1]) else "as grammar"
        SENTENCE = "as path for the sentence" if os.path.isfile(ARGUMENTS[2]) else "as sentence"
        print(f"Using {ARGUMENTS[1]} {GRAMMAR} and {ARGUMENTS[2]} {SENTENCE}.")
        PARSER = Parser(ARGUMENTS[1], ARGUMENTS[2])
        PARSER.parse()
        PARSER.print_tree()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# This file is part of the pyMOR project (http://www.pymor.org).
# Copyright Holders: Rene Milk, Stephan Rave, Felix Schindler
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

'''Thermalblock with GUI demo

Usage:
  thermalblock_gui.py [-h] [--estimator-norm=NORM] [--grid=NI] [--testing]
                  [--help] XBLOCKS YBLOCKS SNAPSHOTS RBSIZE

Arguments:
  XBLOCKS    Number of blocks in x direction.

  YBLOCKS    Number of blocks in y direction.

  SNAPSHOTS  Number of snapshots for basis generation per component.
             In total SNAPSHOTS^(XBLOCKS * YBLOCKS).

  RBSIZE     Size of the reduced basis

Options:
  --estimator-norm=NORM  Norm (trivial, h1) in which to calculate the residual
                         [default: trivial].

  --grid=NI              Use grid with 2*NI*NI elements [default: 60].

  --testing              load the gui and exit right away (for functional testing)

  -h, --help             Show this message.
'''

from __future__ import absolute_import, division, print_function
import sys
from docopt import docopt
import time
from functools import partial
import math as m
import numpy as np
from PySide import QtGui, QtCore
import OpenGL
OpenGL.ERROR_ON_COPY = True

import pymor.core as core
core.logger.MAX_HIERACHY_LEVEL = 2
from pymor.algorithms import greedy, gram_schmidt_basis_extension
from pymor.analyticalproblems import ThermalBlockProblem
from pymor.discretizers import discretize_elliptic_cg
from pymor.gui.glumpy import ColorBarWidget, GlumpyPatchWidget
from pymor.reductors.linear import reduce_stationary_affine_linear

core.getLogger('pymor.algorithms').setLevel('DEBUG')
core.getLogger('pymor.discretizations').setLevel('DEBUG')

PARAM_STEPS = 10
PARAM_MIN = 0.1
PARAM_MAX = 1

class ParamRuler(QtGui.QWidget):
    def __init__(self, parent, sim):
        super(ParamRuler, self).__init__(parent)
        self.sim = sim
        self.setMinimumSize(200, 100)
        box = QtGui.QGridLayout()
        self.spins = []
        for j in xrange(args['YBLOCKS']):
            for i in xrange(args['XBLOCKS']):
                spin = QtGui.QDoubleSpinBox()
                spin.setRange(PARAM_MIN, PARAM_MAX)
                spin.setSingleStep((PARAM_MAX - PARAM_MIN) / PARAM_STEPS)
                spin.setValue(PARAM_MIN)
                self.spins.append(spin)
                box.addWidget(spin, j, i)
                spin.valueChanged.connect(parent.solve_update)
        self.setLayout(box)

    def enable(self, enable=True):
        for spin in self.spins:
            spin.isEnabled = enable

class SimPanel(QtGui.QWidget):
    def __init__(self, parent, sim):
        super(SimPanel, self).__init__(parent)
        self.sim = sim
        box = QtGui.QHBoxLayout()
        self.solution = GlumpyPatchWidget(self, self.sim.grid, vmin=0., vmax=0.8)
        self.bar = ColorBarWidget(self, vmin=0., vmax=0.8)
        box.addWidget(self.solution, 2)
        box.addWidget(self.bar, 2)
        self.param_panel = ParamRuler(self, sim)
        box.addWidget(self.param_panel)
        self.setLayout(box)

    def solve_update(self):
        tic = time.time()
        self.param_panel.enable(False)
        args = self.sim.args
        shape = (args['YBLOCKS'], args['XBLOCKS'])
        mu = {'diffusion': np.array([s.value() for s in self.param_panel.spins]).reshape(shape)}
        U = self.sim.solve(mu)
        print('Simtime {}'.format(time.time() - tic))
        tic = time.time()
        self.solution.set(U.data.ravel())
        self.param_panel.enable(True)
        print('Drawtime {}'.format(time.time() - tic))

class AllPanel(QtGui.QWidget):
    def __init__(self, parent, reduced_sim, detailed_sim):
        super(AllPanel, self).__init__(parent)

        box = QtGui.QVBoxLayout()
        self.reduced_panel = SimPanel(self, reduced_sim)
        self.detailed_panel = SimPanel(self, detailed_sim)
        box.addWidget(self.reduced_panel)
        box.addWidget(self.detailed_panel)
        self.setLayout(box)

class RBGui(QtGui.QMainWindow):
    def __init__(self, args):
        super(RBGui, self).__init__()
        args['XBLOCKS'] = int(args['XBLOCKS'])
        args['YBLOCKS'] = int(args['YBLOCKS'])
        args['--grid'] = int(args['--grid'])
        args['SNAPSHOTS'] = int(args['SNAPSHOTS'])
        args['RBSIZE'] = int(args['RBSIZE'])
        args['--estimator-norm'] = args['--estimator-norm'].lower()
        assert args['--estimator-norm'] in {'trivial', 'h1'}
        reduced = ReducedSim(args)
        detailed = DetailedSim(args)
        self.panel = AllPanel(self, reduced, detailed)
        self.setCentralWidget(self.panel)

class SimBase(object):
    def __init__(self, args):
        self.args = args
        self.first = True
        self.problem = ThermalBlockProblem(num_blocks=(args['XBLOCKS'], args['YBLOCKS']),
                                           parameter_range=(PARAM_MIN, PARAM_MAX))
        self.discretization, pack = discretize_elliptic_cg(self.problem, diameter=m.sqrt(2) / args['--grid'])
        self.grid = pack['grid']

class ReducedSim(SimBase):

    def __init__(self, args):
        super(ReducedSim, self).__init__(args)

    def _first(self):
        args = self.args
        error_product = self.discretization.h1_product if args['--estimator-norm'] == 'h1' else None
        reductor = partial(reduce_stationary_affine_linear, error_product=error_product)
        extension_algorithm = partial(gram_schmidt_basis_extension, product=self.discretization.h1_product)

        greedy_data = greedy(self.discretization, reductor,
                             self.discretization.parameter_space.sample_uniformly(args['SNAPSHOTS']),
                             use_estimator=True, error_norm=self.discretization.h1_norm,
                             extension_algorithm=extension_algorithm, max_extensions=args['RBSIZE'])
        self.rb_discretization, self.reconstructor = greedy_data['reduced_discretization'], greedy_data['reconstructor']
        self.first = False

    def solve(self, mu):
        if self.first:
            self._first()
        return self.reconstructor.reconstruct(self.rb_discretization.solve(mu))

class DetailedSim(SimBase):

    def __init__(self, args):
        super(DetailedSim, self).__init__(args)
        self.discretization.disable_caching()

    def solve(self, mu):
        return self.discretization.solve(mu)

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    args = docopt(__doc__)
    win = RBGui(args)
    win.show()
    testing = args['--testing']
    if not testing:
        sys.exit(app.exec_())

    win.panel.detailed_panel.solve_update()
    win.panel.reduced_panel.solve_update()
    stopTimer = QtCore.QTimer(timeout=app.quit,singleShot=True)
    stopTimer.start(2000)
    app.exec_()

<commit_msg>[tests] try yet another way to cleanly quit TB gui demo<commit_after>#!/usr/bin/env python
# This file is part of the pyMOR project (http://www.pymor.org).
# Copyright Holders: Rene Milk, Stephan Rave, Felix Schindler
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

'''Thermalblock with GUI demo

Usage:
  thermalblock_gui.py [-h] [--estimator-norm=NORM] [--grid=NI] [--testing]
                  [--help] XBLOCKS YBLOCKS SNAPSHOTS RBSIZE

Arguments:
  XBLOCKS    Number of blocks in x direction.

  YBLOCKS    Number of blocks in y direction.

  SNAPSHOTS  Number of snapshots for basis generation per component.
             In total SNAPSHOTS^(XBLOCKS * YBLOCKS).

  RBSIZE     Size of the reduced basis

Options:
  --estimator-norm=NORM  Norm (trivial, h1) in which to calculate the residual
                         [default: trivial].

  --grid=NI              Use grid with 2*NI*NI elements [default: 60].

  --testing              load the gui and exit right away (for functional testing)

  -h, --help             Show this message.
'''

from __future__ import absolute_import, division, print_function
import sys
from docopt import docopt
import time
from functools import partial
import math as m
import numpy as np
from PySide import QtGui, QtCore
import OpenGL

OpenGL.ERROR_ON_COPY = True

import pymor.core as core
core.logger.MAX_HIERACHY_LEVEL = 2
from pymor.algorithms import greedy, gram_schmidt_basis_extension
from pymor.analyticalproblems import ThermalBlockProblem
from pymor.discretizers import discretize_elliptic_cg
from pymor.gui.glumpy import ColorBarWidget, GlumpyPatchWidget
from pymor.reductors.linear import reduce_stationary_affine_linear
from pymor import gui

core.getLogger('pymor.algorithms').setLevel('DEBUG')
core.getLogger('pymor.discretizations').setLevel('DEBUG')

PARAM_STEPS = 10
PARAM_MIN = 0.1
PARAM_MAX = 1

class ParamRuler(QtGui.QWidget):
    def __init__(self, parent, sim):
        super(ParamRuler, self).__init__(parent)
        self.sim = sim
        self.setMinimumSize(200, 100)
        box = QtGui.QGridLayout()
        self.spins = []
        for j in xrange(args['YBLOCKS']):
            for i in xrange(args['XBLOCKS']):
                spin = QtGui.QDoubleSpinBox()
                spin.setRange(PARAM_MIN, PARAM_MAX)
                spin.setSingleStep((PARAM_MAX - PARAM_MIN) / PARAM_STEPS)
                spin.setValue(PARAM_MIN)
                self.spins.append(spin)
                box.addWidget(spin, j, i)
                spin.valueChanged.connect(parent.solve_update)
        self.setLayout(box)

    def enable(self, enable=True):
        for spin in self.spins:
            spin.isEnabled = enable

class SimPanel(QtGui.QWidget):
    def __init__(self, parent, sim):
        super(SimPanel, self).__init__(parent)
        self.sim = sim
        box = QtGui.QHBoxLayout()
        self.solution = GlumpyPatchWidget(self, self.sim.grid, vmin=0., vmax=0.8)
        self.bar = ColorBarWidget(self, vmin=0., vmax=0.8)
        box.addWidget(self.solution, 2)
        box.addWidget(self.bar, 2)
        self.param_panel = ParamRuler(self, sim)
        box.addWidget(self.param_panel)
        self.setLayout(box)

    def solve_update(self):
        tic = time.time()
        self.param_panel.enable(False)
        args = self.sim.args
        shape = (args['YBLOCKS'], args['XBLOCKS'])
        mu = {'diffusion': np.array([s.value() for s in self.param_panel.spins]).reshape(shape)}
        U = self.sim.solve(mu)
        print('Simtime {}'.format(time.time() - tic))
        tic = time.time()
        self.solution.set(U.data.ravel())
        self.param_panel.enable(True)
        print('Drawtime {}'.format(time.time() - tic))

class AllPanel(QtGui.QWidget):
    def __init__(self, parent, reduced_sim, detailed_sim):
        super(AllPanel, self).__init__(parent)

        box = QtGui.QVBoxLayout()
        self.reduced_panel = SimPanel(self, reduced_sim)
        self.detailed_panel = SimPanel(self, detailed_sim)
        box.addWidget(self.reduced_panel)
        box.addWidget(self.detailed_panel)
        self.setLayout(box)

class RBGui(QtGui.QMainWindow):
    def __init__(self, args):
        super(RBGui, self).__init__()
        args['XBLOCKS'] = int(args['XBLOCKS'])
        args['YBLOCKS'] = int(args['YBLOCKS'])
        args['--grid'] = int(args['--grid'])
        args['SNAPSHOTS'] = int(args['SNAPSHOTS'])
        args['RBSIZE'] = int(args['RBSIZE'])
        args['--estimator-norm'] = args['--estimator-norm'].lower()
        assert args['--estimator-norm'] in {'trivial', 'h1'}
        reduced = ReducedSim(args)
        detailed = DetailedSim(args)
        self.panel = AllPanel(self, reduced, detailed)
        self.setCentralWidget(self.panel)

class SimBase(object):
    def __init__(self, args):
        self.args = args
        self.first = True
        self.problem = ThermalBlockProblem(num_blocks=(args['XBLOCKS'], args['YBLOCKS']),
                                           parameter_range=(PARAM_MIN, PARAM_MAX))
        self.discretization, pack = discretize_elliptic_cg(self.problem, diameter=m.sqrt(2) / args['--grid'])
        self.grid = pack['grid']

class ReducedSim(SimBase):

    def __init__(self, args):
        super(ReducedSim, self).__init__(args)

    def _first(self):
        args = self.args
        error_product = self.discretization.h1_product if args['--estimator-norm'] == 'h1' else None
        reductor = partial(reduce_stationary_affine_linear, error_product=error_product)
        extension_algorithm = partial(gram_schmidt_basis_extension, product=self.discretization.h1_product)

        greedy_data = greedy(self.discretization, reductor,
                             self.discretization.parameter_space.sample_uniformly(args['SNAPSHOTS']),
                             use_estimator=True, error_norm=self.discretization.h1_norm,
                             extension_algorithm=extension_algorithm, max_extensions=args['RBSIZE'])
        self.rb_discretization, self.reconstructor = greedy_data['reduced_discretization'], greedy_data['reconstructor']
        self.first = False

    def solve(self, mu):
        if self.first:
            self._first()
        return self.reconstructor.reconstruct(self.rb_discretization.solve(mu))

class DetailedSim(SimBase):

    def __init__(self, args):
        super(DetailedSim, self).__init__(args)
        self.discretization.disable_caching()

    def solve(self, mu):
        return self.discretization.solve(mu)

if __name__ == '__main__':
    args = docopt(__doc__)
    testing = args['--testing']
    if not testing:
        app = QtGui.QApplication(sys.argv)
        win = RBGui(args)
        win.show()
        sys.exit(app.exec_())

    gui.qt.launch_qt_app(lambda _: RBGui(args), block=False)
<|endoftext|>


<commit_before># coding: utf-8
import re

import lxml.html
from scrapelib import urlopen

from pupa.scrape import Scraper, Jurisdiction, Legislator
from pupa.models.person import Person

CONTACT_DETAIL_TYPE_MAP = {
  'Address': 'address',
  'bb': 'cell',
  'bus': 'voice',
  'Bus': 'voice',
  'Bus.': 'voice',
  'Business': 'voice',
  'Cell': 'cell',
  'Cell Phone': 'cell',
  'Email': 'email',
  'Fax': 'fax',
  'Home': 'voice',
  'Home Phone': 'voice',
  'Home Phone*': 'voice',
  'Office': 'voice',
  'ph': 'voice',
  u'ph\xc2': 'voice',
  'Phone': 'voice',
  'Res': 'voice',
  'Res/Bus': 'voice',
  'Residence': 'voice',
  'Voice Mail': 'voice',
}
# In Newmarket, for example, there are both "Phone" and "Business" numbers.
CONTACT_DETAIL_NOTE_MAP = {
  'Address': 'legislature',
  'bb': 'legislature',
  'bus': 'office',
  'Bus': 'office',
  'Bus.': 'office',
  'Business': 'office',
  'Cell': 'legislature',
  'Cell Phone': 'legislature',
  'Email': None,
  'Fax': 'legislature',
  'Home': 'residence',
  'Home Phone': 'residence',
  'Home Phone*': 'residence',
  'ph': 'legislature',
  u'ph\xc2': 'legislature',
  'Phone': 'legislature',
  'Office': 'legislature',
  'Res': 'residence',
  'Res/Bus': 'office',
  'Residence': 'residence',
  'Voice Mail': 'legislature',
}

class CanadianJurisdiction(Jurisdiction):
  session_details = {
    'N/A': {
      '_scraped_name': 'N/A',
    }
  }
  terms = [
    {
      'name': 'N/A',
      'sessions': ['N/A'],
      'start_year': 1900,
      'end_year': 2030,
    }
  ]

  def __init__(self):
    for scraper_type in ('bills', 'events', 'people', 'speeches', 'votes'):
      try:
        __import__(self.__module__ + '.' + scraper_type)
      except ImportError:
        pass
      else:
        self.provides.append(scraper_type)

  def get_scraper(self, term, session, scraper_type):
    if scraper_type in self.provides:
      class_name = self.__class__.__name__ + {
        'bills': 'Bill',
        'events': 'Event',
        'people': 'Person',
        'speeches': 'Speech',
        'votes': 'Vote',
      }[scraper_type] + 'Scraper'
      return getattr(__import__(self.__module__ + '.' + scraper_type, fromlist=[class_name]), class_name)

  def scrape_session_list(self):
    return ['N/A']

class CanadianLegislator(Legislator):
    def __init__(self, name, post_id, **kwargs):
      super(CanadianLegislator, self).__init__(clean_name(name), clean_post_id(post_id), **kwargs)

    def add_link(self, url, note=None):
        if url.startswith('www.'):
          url = 'http://%s' % url
        if re.match('\A@[A-Za-z]+\Z', url):
          url = 'https://twitter.com/%s' % url[1:]
        self.links.append({"note": note, "url": url})

# Removes _is_legislator flag, _contact_details and _role. Used by aggregations.
# @see https://github.com/opencivicdata/pupa/blob/master/pupa/scrape/helpers.py
class AggregationLegislator(Person):
  __slots__ = ('post_id', 'party', 'chamber')

  def __init__(self, name, post_id, party=None, chamber=None, **kwargs):
    super(AggregationLegislator, self).__init__(clean_name(name), **kwargs)
    self.post_id = clean_post_id(post_id)
    self.party = party
    self.chamber = chamber

def clean_name(value):
  return value.replace(u' ', ' ').replace(u'’', "'").replace('Mayor', '').replace('Councillor', '').strip()

def clean_post_id(name):
  return value.replace(u' ', ' ').replace(u'’', "'") # non-breaking space

def lxmlize(url, encoding='utf-8'):
  entry = urlopen(url).encode(encoding)
  page = lxml.html.fromstring(entry)
  meta = page.xpath('//meta[@http-equiv="refresh"]')
  if meta:
    _, url = meta[0].attrib['content'].split('=', 1)
    return lxmlize(url, encoding)
  else:
    page.make_links_absolute(url)
    return page
<commit_msg>fix variable name<commit_after># coding: utf-8
import re

import lxml.html
from scrapelib import urlopen

from pupa.scrape import Scraper, Jurisdiction, Legislator
from pupa.models.person import Person

CONTACT_DETAIL_TYPE_MAP = {
  'Address': 'address',
  'bb': 'cell',
  'bus': 'voice',
  'Bus': 'voice',
  'Bus.': 'voice',
  'Business': 'voice',
  'Cell': 'cell',
  'Cell Phone': 'cell',
  'Email': 'email',
  'Fax': 'fax',
  'Home': 'voice',
  'Home Phone': 'voice',
  'Home Phone*': 'voice',
  'Office': 'voice',
  'ph': 'voice',
  u'ph\xc2': 'voice',
  'Phone': 'voice',
  'Res': 'voice',
  'Res/Bus': 'voice',
  'Residence': 'voice',
  'Voice Mail': 'voice',
}
# In Newmarket, for example, there are both "Phone" and "Business" numbers.
CONTACT_DETAIL_NOTE_MAP = {
  'Address': 'legislature',
  'bb': 'legislature',
  'bus': 'office',
  'Bus': 'office',
  'Bus.': 'office',
  'Business': 'office',
  'Cell': 'legislature',
  'Cell Phone': 'legislature',
  'Email': None,
  'Fax': 'legislature',
  'Home': 'residence',
  'Home Phone': 'residence',
  'Home Phone*': 'residence',
  'ph': 'legislature',
  u'ph\xc2': 'legislature',
  'Phone': 'legislature',
  'Office': 'legislature',
  'Res': 'residence',
  'Res/Bus': 'office',
  'Residence': 'residence',
  'Voice Mail': 'legislature',
}

class CanadianJurisdiction(Jurisdiction):
  session_details = {
    'N/A': {
      '_scraped_name': 'N/A',
    }
  }
  terms = [
    {
      'name': 'N/A',
      'sessions': ['N/A'],
      'start_year': 1900,
      'end_year': 2030,
    }
  ]

  def __init__(self):
    for scraper_type in ('bills', 'events', 'people', 'speeches', 'votes'):
      try:
        __import__(self.__module__ + '.' + scraper_type)
      except ImportError:
        pass
      else:
        self.provides.append(scraper_type)

  def get_scraper(self, term, session, scraper_type):
    if scraper_type in self.provides:
      class_name = self.__class__.__name__ + {
        'bills': 'Bill',
        'events': 'Event',
        'people': 'Person',
        'speeches': 'Speech',
        'votes': 'Vote',
      }[scraper_type] + 'Scraper'
      return getattr(__import__(self.__module__ + '.' + scraper_type, fromlist=[class_name]), class_name)

  def scrape_session_list(self):
    return ['N/A']

class CanadianLegislator(Legislator):
    def __init__(self, name, post_id, **kwargs):
      super(CanadianLegislator, self).__init__(clean_name(name), clean_post_id(post_id), **kwargs)

    def add_link(self, url, note=None):
        if url.startswith('www.'):
          url = 'http://%s' % url
        if re.match('\A@[A-Za-z]+\Z', url):
          url = 'https://twitter.com/%s' % url[1:]
        self.links.append({"note": note, "url": url})

# Removes _is_legislator flag, _contact_details and _role. Used by aggregations.
# @see https://github.com/opencivicdata/pupa/blob/master/pupa/scrape/helpers.py
class AggregationLegislator(Person):
  __slots__ = ('post_id', 'party', 'chamber')

  def __init__(self, name, post_id, party=None, chamber=None, **kwargs):
    super(AggregationLegislator, self).__init__(clean_name(name), **kwargs)
    self.post_id = clean_post_id(post_id)
    self.party = party
    self.chamber = chamber

def clean_name(value):
  return value.replace(u' ', ' ').replace(u'’', "'").replace('Mayor', '').replace('Councillor', '').strip()

def clean_post_id(value):
  return value.replace(u' ', ' ').replace(u'’', "'") # non-breaking space

def lxmlize(url, encoding='utf-8'):
  entry = urlopen(url).encode(encoding)
  page = lxml.html.fromstring(entry)
  meta = page.xpath('//meta[@http-equiv="refresh"]')
  if meta:
    _, url = meta[0].attrib['content'].split('=', 1)
    return lxmlize(url, encoding)
  else:
    page.make_links_absolute(url)
    return page
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import time
import os.path

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from timer import TimerThread

from lock import with_threading_lock

class DataFile(object):

    STATUS_NOT_FOUND = "not_found"
    STATUS_LOADED = "loaded"
    STATUS_UPDATED = "updated"

    def __init__(self, name, path):
        self.name = name
        self.path = path
        self.status = self.STATUS_NOT_FOUND
        self.data = None

    def is_ready(self):
        return self.status == self.STATUS_UPDATED

    def on_data(self, path):
        self.status = self.STATUS_UPDATED

        # TODO: parse

    def on_parsed(self):
        self.status == self.STATUS_LOADED

class Profile(object):

    ALL_DATA = [
        "ContractSystem.txt",
        "Funding.txt",
        "PCScenario.txt",
        "ProgressTracking.txt",
        "Reputation.txt",
        "ResearchAndDevelopment.txt",
        "ResourceScenario.txt",
        "ScenarioDestructibles.txt",
        "ScenarioNewGameIntro.txt",
        "ScenarioUpgradeableFacilities.txt",
        "StrategySystem.txt",
        "VesselRecovery.txt",
    ]

    FILE_TIMEOUT = 4

    def __init__(self, name, base_path):
        print "New Profile: ", name
        self.name = name
        self.base_path = base_path

        self.data = {}
        for name in self.ALL_DATA:
            self.data[name] = DataFile(name, os.path.join(self.base_path, name))

        self.timer = TimerThread(self.parse_files)
        self.timer.start()

    def stop(self):
        self.timer.stop()

        # TODO: should we do one last merge?
        self.parse_files()

    @with_threading_lock('data_files')
    def update_file(self, path):
        if path not in self.data:
            return

        print self.name, "updated", path

        self.data[path].on_data(os.path.join(self.base_path, self.name, path))

        # Waits for the timer to finish
        if not self.timer.is_on():
            self.timer.set(self.FILE_TIMEOUT)

    @with_threading_lock('data_files')
    def parse_files(self):
        print "Running Merger"

class ProfileHandler(FileSystemEventHandler):

    def __init__(self, base_path):
        super(ProfileHandler, self).__init__()

        self.profiles = {}
        self.base_path = base_path

        # Scan the folder on init
        self.load_profiles()

    def load_profiles(self):
        '''
        Loads the profiles and their files that currently exist in the system
        '''
        folders = [f for f in os.listdir(self.base_path) if not os.path.isfile(os.path.join(self.base_path, f))]

        for folder in folders:
            self.profiles[folder] = Profile(folder, self.base_path)

            path = os.path.join(self.base_path, folder)

            files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(self.base_path, folder, f))]
            for file in files:
                self.profiles[folder].update_file(file)

    def on_created(self, event):
        '''
        Listen for new profiles being created.
        Listen for new files being added to a profile
        '''
        path = os.path.relpath(event.src_path, self.base_path)
        parts = path.split("/")

        name = parts[0]

        if len(parts) == 1:
            self.profiles[name] = Profile(name, self.base_path)
        elif len(parts) == 2:
            self.profiles[name].update_file(parts[1])
        else:
            print "Warning: Subdirectories have been created"

    def on_modified(self, event):
        '''
        Listen for file modifications under a profile
        '''
        path = os.path.relpath(event.src_path, self.base_path)
        parts = path.split("/")

        name = parts[0]

        if len(parts) == 1:
            # This might mean a data file has been created
            # This might mean a data file has been removed
            # Therefore ignore it
            pass
        elif len(parts) == 2:
            self.profiles[name].update_file(parts[1])
        else:
            print "Warning: Subdirectories have been edited"

    def stop(self):
        for key, profile in self.profiles.iteritems():
            profile.stop()

if __name__ == "__main__":
    path = '/Users/Saevon/Applications/Kerbal Space Program/DMPServer/Universe/Scenarios'
    observer = Observer()

    profile_handler = ProfileHandler(path)
    observer.schedule(profile_handler, path, recursive=True)
    observer.start()

    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        profile_handler.stop()
        observer.stop()
    observer.join()
<commit_msg>Adds parser to the watcher<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import time
import os.path

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from timer import TimerThread

from lock import with_threading_lock

import parser

class DataFile(object):

    STATUS_NOT_FOUND = "not_found"
    STATUS_LOADED = "loaded"
    STATUS_UPDATED = "updated"

    def __init__(self, name, path):
        self.name = name
        self.path = path
        self.status = self.STATUS_NOT_FOUND
        self.data = None

    def is_ready(self):
        return self.status == self.STATUS_UPDATED

    def on_data(self, path):
        self.status = self.STATUS_UPDATED

        with open(self.path, 'r') as fp:
            self.data = parser.load(fp)

    def on_parsed(self):
        self.status == self.STATUS_LOADED

class Profile(object):

    ALL_DATA = [
        "ContractSystem.txt",
        "Funding.txt",
        "PCScenario.txt",
        "ProgressTracking.txt",
        "Reputation.txt",
        "ResearchAndDevelopment.txt",
        "ResourceScenario.txt",
        "ScenarioDestructibles.txt",
        "ScenarioNewGameIntro.txt",
        "ScenarioUpgradeableFacilities.txt",
        "StrategySystem.txt",
        "VesselRecovery.txt",
    ]

    FILE_TIMEOUT = 4

    def __init__(self, name, base_path):
        print "New Profile: ", name
        self.name = name
        self.base_path = base_path

        self.data = {}
        for name in self.ALL_DATA:
            self.data[name] = DataFile(name, os.path.join(self.base_path, name))

        self.timer = TimerThread(self.parse_files)
        self.timer.start()

    def stop(self):
        self.timer.stop()

        # TODO: should we do one last merge?
        self.parse_files()

    @with_threading_lock('data_files')
    def update_file(self, path):
        if path not in self.data:
            return

        print self.name, "updated", path

        self.data[path].on_data(os.path.join(self.base_path, self.name, path))

        # Waits for the timer to finish
        if not self.timer.is_on():
            self.timer.set(self.FILE_TIMEOUT)

    @with_threading_lock('data_files')
    def parse_files(self):
        print "Running Merger"

class ProfileHandler(FileSystemEventHandler):

    def __init__(self, base_path):
        super(ProfileHandler, self).__init__()

        self.profiles = {}
        self.base_path = base_path

        # Scan the folder on init
        self.load_profiles()

    def load_profiles(self):
        '''
        Loads the profiles and their files that currently exist in the system
        '''
        folders = [f for f in os.listdir(self.base_path) if not os.path.isfile(os.path.join(self.base_path, f))]

        for folder in folders:
            self.profiles[folder] = Profile(folder, self.base_path)

            path = os.path.join(self.base_path, folder)

            files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(self.base_path, folder, f))]
            for file in files:
                self.profiles[folder].update_file(file)

    def on_created(self, event):
        '''
        Listen for new profiles being created.
        Listen for new files being added to a profile
        '''
        path = os.path.relpath(event.src_path, self.base_path)
        parts = path.split("/")

        name = parts[0]

        if len(parts) == 1:
            self.profiles[name] = Profile(name, self.base_path)
        elif len(parts) == 2:
            self.profiles[name].update_file(parts[1])
        else:
            print "Warning: Subdirectories have been created"

    def on_modified(self, event):
        '''
        Listen for file modifications under a profile
        '''
        path = os.path.relpath(event.src_path, self.base_path)
        parts = path.split("/")

        name = parts[0]

        if len(parts) == 1:
            # This might mean a data file has been created
            # This might mean a data file has been removed
            # Therefore ignore it
            pass
        elif len(parts) == 2:
            self.profiles[name].update_file(parts[1])
        else:
            print "Warning: Subdirectories have been edited"

    def stop(self):
        for key, profile in self.profiles.iteritems():
            profile.stop()

if __name__ == "__main__":
    path = '/Users/Saevon/Applications/Kerbal Space Program/DMPServer/Universe/Scenarios'
    observer = Observer()

    profile_handler = ProfileHandler(path)
    observer.schedule(profile_handler, path, recursive=True)
    observer.start()

    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        profile_handler.stop()
        observer.stop()
    observer.join()
<|endoftext|>


<commit_before>'''Converts a cURL command to code for Python Requests'''
import curl_to_requests.curl_to_requests as curl_to_requests

import webapi
class WebAPI(webapi.WebAPI):
    input_text = webapi.text_input('Input')
    submit_button = webapi.submit_button('Convert')

    def run(self, form_input):
        input_text = form_input['input_text'].strip()
        input_text = input_text.replace('\\\r\n','')
        input_text = input_text.replace('\\\r','')
        output = curl_to_requests.curl_to_requests(input_text)

        return {'output_type' : 'simple',
                'output' : output}
<commit_msg>changed name<commit_after>'''Converts a cURL command to code for Python Requests'''
import curl2requests.curl_to_requests as curl_to_requests

import webapi
class WebAPI(webapi.WebAPI):
    input_text = webapi.text_input('Input')
    submit_button = webapi.submit_button('Convert')

    def run(self, form_input):
        input_text = form_input['input_text'].strip()
        input_text = input_text.replace('\\\r\n','')
        input_text = input_text.replace('\\\r','')
        output = curl_to_requests.curl_to_requests(input_text)

        return {'output_type' : 'simple',
                'output' : output}
<|endoftext|>


<commit_before>#from allennlp.predictors.predictor import Predictor
import json
from tqdm import tqdm
import re

def perform_srl():

    predictor = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/srl-model-2018.05.25.tar.gz")
    sentence_start = "Schools should not allow junk food to be sold on campus "

    sentences = []
    with open("sentences.txt") as i:
      for line in i:
        line = line.strip()
        sentence = sentence_start + line
        sentences.append(sentence)

    # TODO: Prediction is much more efficient if we feed the sentences as a batch.
    output = []
    for sentence in tqdm(sentences):
        x = predictor.predict(sentence)
        output.append(x)

    with open("output.json", "w") as o:
        json.dump(output, o)

def parse_srl(f):

    modal_verbs = set(["will", "shall", "may", "might", "can", "could", "must", "ought to",
                       "should", "would", "used to", "need"])

    aux_verbs = set(["do", "does"])

    all_sentences = []
    with open(f) as i:
        d = json.load(i)
        for sent in d:
            sentence_info = {"sentence": " ".join(sent["words"][11:]),
                             "meta": [],
                             "verbs": []}

            auxiliary = None
            for verb in sent["verbs"]:
                verb_string = verb["verb"].lower()

                if verb_string in modal_verbs:
                    sentence_info["meta"].append("modal")
                    auxiliary = verb_string

                elif verb_string in aux_verbs:
                    sentence_info["meta"].append("auxiliary")
                    auxiliary = verb_string

                else:
                    arg0_string = "-"
                    arg1_string = "-"
                    arg2_string = "-"

                    arg0 = re.search("\[ARG0: (.*?)\]", verb["description"])
                    if arg0:
                        arg0_string = arg0.group(1)
                    arg1 = re.search("\[ARG1: (.*?)\]", verb["description"])
                    if arg1:
                        arg1_string = arg1.group(1)

                    arg2 = re.search("\[ARG2: (.*?)\]", verb["description"])
                    if arg2:
                        arg2_string = arg2.group(1)

                    if auxiliary is not None:
                        verb_string += f" ({auxiliary})"
                        auxiliary = None

                    sentence_info["verbs"].append([verb_string, arg0_string, arg1_string, arg2_string])

            all_sentences.append(sentence_info)

    for sentence in all_sentences:
        output_list = [sentence["sentence"], " ".join(sentence["meta"][1:])]  # strip the first "meta" label: main clause
        for verb in sentence["verbs"]:
            for item in verb:
                output_list.append(item)

        print("\t".join(output_list))

#perform_srl()
parse_srl("data/srl_output.json")
<commit_msg>Added batch processing of input sentences<commit_after>from allennlp.predictors.predictor import Predictor
import json
from tqdm import tqdm
import re

def perform_srl(f, sentence_start):

    predictor = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/srl-model-2018.05.25.tar.gz")

    sentences, responses = [], []
    with open(f) as i:
      for line in i:
        line = line.strip()
        sentence = sentence_start + line
        responses.append(line)
        sentences.append({"sentence": sentence})

    output = predictor.predict_batch_json(sentences)

    full_output = [{"sentence": sentence_start + response,
                    "response": response,
                    "srl": srl} for (response, srl) in zip(responses, output)]

    return full_output

def identify_argument(verb, argument_number):
    arg0 = re.search(f"\[ARG{argument_number}: (.*?)\]", verb["description"])
    if arg0:
        return arg0.group(1)
    return "-"

def parse_srl_output(srl_output_list, sentence_start):

    modal_verbs = set(["will", "shall", "may", "might", "can", "could", "must", "ought to",
                       "should", "would", "used to", "need"])

    aux_verbs = set(["do", "does"])

    all_sentences = []
    for sent in srl_output_list:
        sentence_info = {"sentence": sent["sentence"],
                         "response": sent["response"],
                         "meta": [],
                         "verbs": []}

        auxiliary = None
        for verb in sent["srl"]["verbs"]:
            verb_string = verb["verb"].lower()

            if verb_string in modal_verbs:
                sentence_info["meta"].append("modal")
                auxiliary = verb_string

            elif verb_string in aux_verbs:
                sentence_info["meta"].append("auxiliary")
                auxiliary = verb_string

            else:
                arg0_string = identify_argument(verb, 0)
                arg1_string = identify_argument(verb, 1)
                arg2_string = identify_argument(verb, 2)

                if auxiliary is not None:
                    verb_string += f" ({auxiliary})"
                    auxiliary = None

                sentence_info["verbs"].append([verb_string, arg0_string, arg1_string, arg2_string])

        all_sentences.append(sentence_info)

    all_sentences.sort(key = lambda x: x["verbs"][3][0] if len(x["verbs"]) > 3 else "")

    for sentence in all_sentences:
        output_list = [sentence["response"], " ".join(sentence["meta"][1:])]  # strip the first "meta" label: main clause
        for verb in sentence["verbs"][3:]:
            for item in verb:
                output_list.append(item)

        print("\t".join(output_list))

sentence_start = "Schools should not allow junk food to be sold on campus "

srl_out = perform_srl("scripts/data/sentences.txt", sentence_start)
parse_srl_output(srl_out, sentence_start)
<|endoftext|>


<commit_before># Python Standard Library Imports
import Adafruit_BBIO.UART as UART
from Adafruit_I2C import Adafruit_I2C
import Adafruit_BBIO.GPIO as GPIO
import serial
import struct
import time
import math
from time import sleep
import datetime
import mpu6050
import Sabertooth
import Kalman
import BatteryMonitor
import Controllers

# Start the UART1
UART.setup("UART1")

# Configuration of the GPIO's
RedLED="P8_13"
GreenLED="P9_27"
ResetButton="P8_15"

GPIO.setup(ResetButton, GPIO.IN)
GPIO.setup(RedLED, GPIO.OUT)
GPIO.setup(GreenLED, GPIO.OUT)

# MPU6050 initialization
mpu = mpu6050.MPU6050()
#Test connection
if mpu.getDeviceID()!=52:
	GPIO.output(RedLED, GPIO.HIGH)
else:
	mpu.dmpInitialize()
	mpu.setDMPEnabled(True)
	# get expected DMP packet size for later comparison
	packetSize = mpu.dmpGetFIFOPacketSize()

#Global Variables
global AccXangle
global GYRx
global ax_average
global GYRx_average
global PI

# Variable initialization
AccXangle=0
GYRx=0
ax_average=0
GYRx_average=0
PI = 3.14159265358979323846

#Initialization of class from the others files
PC=Sabertooth.PacketizedCommunication()
KF=Kalman.KalmanFilter()
Battery=BatteryMonitor.BatteryValue()
PID=Controllers.PIDControllers()

# Calibration of the values from the sensor MPU6050 (vertical position)
def MPU6050_Calibration():

	numReadings=100							# Number of readings for calibration
	ax_total=0
	GYRx_total=0
	global ax_average
	global GYRx_average

	while (AccXangle<-1 or AccXangle>1):				# Keep angle between these values
		MPU6050_Values()					# Call function to get the values

	for i in range(0,numReadings):
		ax_total=AccXangle+ax_total
		GYRx_total=GYRx+GYRx_total

	ax_average=float (ax_total)/numReadings			        # Average accelerometer value
	GYRx_average=float (GYRx_total)/numReadings			# Average gyroscope value

	return ax_average, GYRx_average					# Returns the offset

def MPU6050_Values():
	global AccXangle
	global GYRx
	global PI
	RAD_TO_DEG = 57.29578						# Convertion rate, Rad to Deg

	GYRx=mpu.readGYRx()						# Reading of variables of interest
	accZ=mpu.readACCz()
	accY=mpu.readACCy()

	AccXangle = (math.atan2(accY,accZ)+PI)*RAD_TO_DEG		# Calculation of the angle in X axis
	AccXangle-=180							# Correction of angle (depends on the position of the sensor)

	return (GYRx, AccXangle)

PC.set_baud(Sabertooth.addr,Sabertooth.baud)
sleep(3)								# Wait to stabilize the communication
MPU6050_Values()							# Make a reading from the MPU6050 to get in the while () condition from the calibration
while MPU6050_Calibration():						# Calibration routine
	PC.stopAndReset()						# Initialize at zero
GPIO.output(GreenLED, GPIO.HIGH)					# Led state to green
sleep(1.5)

while True:
	LoopTime = time.time()						# Start the time stamp for the loop

	if Battery.BatteryVoltage()<10.5:				# Case low battery, stop robot
		PC.stopAndReset()
	else:
		#Read of the MPU6050 values
		MPU6050_Values()

		#Calculate the new x angle with the ofset of the Calibration
		ax_raw=AccXangle-ax_average				# Calculate the values from MPU6050 to send to the Kalman filter
		GYRx_raw=GYRx-GYRx_average

		x_angle=KF.KalmanCalculate(ax_raw,GYRx_raw,LoopTime)	# Get filtered value in function of acceleration, angle and looptime

		PiVelocityRef=PID.PiPosition()				# Calculate VelocityRef based on position
		PiAngleRef=PID.PiVelocity(PiVelocityRef)		# Calculate AngleRef based on velocity
		PidMotorRef=PID.PidAngle(x_angle,PiAngleRef)		# Calculate MotorRef based on the angle

		PC.drive(Sabertooth.addr, 1, int(PidMotorRef))		# Value sent to motors
		PC.drive(Sabertooth.addr, 2, int(PidMotorRef))

<commit_msg>Update ProBot.py<commit_after># Python Standard Library Imports
import Adafruit_BBIO.UART as UART
from Adafruit_I2C import Adafruit_I2C
import Adafruit_BBIO.GPIO as GPIO
import serial
import struct
import time
import math
from time import sleep
import datetime
import mpu6050
import Sabertooth
import Kalman
import BatteryMonitor
import Controllers

# Start the UART1
UART.setup("UART1")

# Configuration of the GPIO's
RedLED="P8_13"
GreenLED="P9_27"
ResetButton="P8_15"

GPIO.setup(ResetButton, GPIO.IN)
GPIO.setup(RedLED, GPIO.OUT)
GPIO.setup(GreenLED, GPIO.OUT)

# MPU6050 initialization
mpu = mpu6050.MPU6050()
#Test connection
if mpu.getDeviceID()!=52:
	GPIO.output(RedLED, GPIO.HIGH)
else:
	mpu.dmpInitialize()
	mpu.setDMPEnabled(True)
	# get expected DMP packet size for later comparison
	packetSize = mpu.dmpGetFIFOPacketSize()

#Global Variables
global AccXangle
global GYRx
global ax_average
global GYRx_average
global PI

# Variable initialization
AccXangle=0
GYRx=0
ax_average=0
GYRx_average=0
PI = 3.14159265358979323846

#Initialization of class from the others files
PC=Sabertooth.PacketizedCommunication()
KF=Kalman.KalmanFilter()
Battery=BatteryMonitor.BatteryValue()
PID=Controllers.PIDControllers()

# Calibration of the values from the sensor MPU6050 (vertical position)
def MPU6050_Calibration():

	numReadings=100							# Number of readings for calibration
	ax_total=0
	GYRx_total=0
	global ax_average
	global GYRx_average

	while (AccXangle<-1 or AccXangle>1):				# Keep angle between these values
		MPU6050_Values()					# Call function to get the values

	for i in range(0,numReadings):
		ax_total=AccXangle+ax_total
		GYRx_total=GYRx+GYRx_total

	ax_average=float (ax_total)/numReadings			        # Average accelerometer value
	GYRx_average=float (GYRx_total)/numReadings			# Average gyroscope value

	return ax_average, GYRx_average					# Returns the offset

def MPU6050_Values():
	global AccXangle
	global GYRx
	global PI
	RAD_TO_DEG = 57.29578						# Convertion rate, Rad to Deg

	GYRx=mpu.readGYRx()						# Reading of variables of interest
	accZ=mpu.readACCz()
	accY=mpu.readACCy()

	AccXangle = (math.atan2(accY,accZ)+PI)*RAD_TO_DEG		# Calculation of the angle in X axis
	AccXangle-=180							# Correction of angle (depends on the position of the sensor)

	return (GYRx, AccXangle)

PC.set_baud(Sabertooth.addr,Sabertooth.baud)
sleep(3)								# Wait to stabilize the communication
MPU6050_Values()							# Make a reading from the MPU6050 to get in the while () condition from the calibration
while MPU6050_Calibration():						# Calibration routine
	PC.stopAndReset()						# Keep at zero the motors
GPIO.output(GreenLED, GPIO.HIGH)					# Led state to green
sleep(1.5)

while True:
	LoopTime = time.time()						# Start the time stamp for the loop

	if Battery.BatteryVoltage()<10.5:				# Case low battery, stop robot
		PC.stopAndReset()
	else:
		#Read of the MPU6050 values
		MPU6050_Values()

		#Calculate the new x angle with the ofset of the Calibration
		ax_raw=AccXangle-ax_average				# Calculate the values from MPU6050 to send to the Kalman filter
		GYRx_raw=GYRx-GYRx_average

		x_angle=KF.KalmanCalculate(ax_raw,GYRx_raw,LoopTime)	# Get filtered value in function of acceleration, angle and looptime

		PiVelocityRef=PID.PiPosition()				# Calculate VelocityRef based on position
		PiAngleRef=PID.PiVelocity(PiVelocityRef)		# Calculate AngleRef based on velocity
		PidMotorRef=PID.PidAngle(x_angle,PiAngleRef)		# Calculate MotorRef based on the angle

		PC.drive(Sabertooth.addr, 1, int(PidMotorRef))		# Value sent to motors
		PC.drive(Sabertooth.addr, 2, int(PidMotorRef))

<|endoftext|>