<commit_before># -*- coding: utf-8 -*-

"""
MIT License

Copyright (c) 2018 Samuel Riches

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import datetime
import ipaddress
import logging

from .country_codes import codes

log = logging.getLogger(__name__)

# single IP lookup

async def fetch_ip_info(address, *, conn, session, token):
    """
    Fetch information about a single IP Address from the database.
    If the entry exists it's lookup counter will be increased.

    Parameters
    ----------
    address : str
        The IP Address to look up.
    conn : Union[asyncpg.Connection, asyncpg.pool.Pool]
        The connection to the database.
    session : aiohttp.ClientSession
        The session to fetch address info with if it's not cached.
    token : str
        The API token to look up addresses.
    """

    info = await fetch_cached_info(address, conn=conn)

    if info is not None:
        if (datetime.datetime.now() - info.get("last_updated")) > datetime.timedelta(days=7):
            log.info(f'IP info for address "{address}" old, updating ..')
            info = await update_full_address_info(address, conn=conn, session=session, token=token)
        else:
            log.info(f'IP info for address "{address}" cached.')
    else:
        log.info(f'IP info for address "{address}" not cached, fetching ..')
        info = await fetch_address_info(address, conn=conn, session=session, token=token)
    try:
        return format_response(info)
    except KeyError as KE:
        log.info("KeyError {}".format(KE))
        info = await update_address_info(address, conn=conn, session=session, token=token)
        log.info("Updated loc for {}".format(address))
        return format_response(info)

async def fetch_cached_info(address, *, conn):
    record = await conn.fetchrow(
        'SELECT ip, country_code, accessed, city, region, org, last_updated, loc FROM ip_storage WHERE ip = $1', address
    )

    if record is None:
        return None

    await conn.execute('UPDATE ip_storage SET accessed = ip_storage.accessed + 1 WHERE ip = $1', address)
    return dict(record)

async def update_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        'UPDATE ip_storage SET loc = $2 WHERE ip = $1',
        address,
        loc
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

async def update_full_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        """
        UPDATE ip_storage
        SET country_code = $1,
        city = $2,
        region = $3,
        org = $4,
        loc = $5
        WHERE ip = $6
        """,
        country_code, city, region, org, loc, address
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

async def fetch_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        'INSERT INTO ip_storage(ip, country_code, city, region, org, loc) VALUES ($1, $2, $3, $4, $5, $6)',
        address,
        country_code,
        city,
        region,
        org,
        loc
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

def format_response(info):
    ip = info['ip']
    country_code = info['country_code']

    response = {
        'global': ip.is_global,
        'loopback': ip.is_loopback,
        'max_prefixlen': ip.max_prefixlen,
        'multicast': ip.is_multicast,
        'link_local': ip.is_link_local,
        'private': ip.is_private,
        'reserved': ip.is_reserved,
        'reverse_pointer': ip.reverse_pointer,
        'unspecified': ip.is_unspecified,
        'version': ip.version,

        'city': info['city'],
        'country': codes.get(country_code, '??'),
        'country_code': country_code,
        'region': info['region'],
        'org': info['org'],
        'loc': info['loc']
    }

    if isinstance(ip, ipaddress.IPv6Address):
        response.update({
            'ipv4_mapped': ip.ipv4_mapped,
            'site_local': ip.is_site_local,
            'sixtofour': ip.sixtofour,
            'teredo': ip.teredo,
        })

    return response

# API statistics

async def fetch_lookup_stats(countries, *, conn):
    """
    Fetch how many times all, a single or multiple countries have been looked up via the API.

    Parameters
    ----------
    countries : Optional[List[str]]
        The countries to look up. If None all countries will be looked up.
    conn : Union[asyncpg.Connection, asyncpg.pool.Pool]
        The connection to the database.
    """

    if countries is not None:
        records = await conn.fetch(
            """
            SELECT count(*) AS stored_ips, CAST (sum(accessed) AS INTEGER) AS lookups, country_code
            FROM ip_storage
            WHERE country_code = ANY($1::TEXT[])
            GROUP BY country_code
            """,
            countries,
        )
    else:
        records = await conn.fetch(
            """
            SELECT count(*) AS stored_ips, CAST (sum(accessed) AS INTEGER) AS lookups, country_code
            FROM ip_storage
            GROUP BY country_code
            """
        )

    results = {
        x['country_code']: {
            'country': codes.get(x['country_code'], '??'), 'lookups': x['lookups'], 'stored_ips': x['stored_ips']
        } for x in records
    }

    return results
<commit_msg>Fix not updating<commit_after># -*- coding: utf-8 -*-

"""
MIT License

Copyright (c) 2018 Samuel Riches

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import datetime
import ipaddress
import logging

from .country_codes import codes

log = logging.getLogger(__name__)

# single IP lookup

async def fetch_ip_info(address, *, conn, session, token):
    """
    Fetch information about a single IP Address from the database.
    If the entry exists it's lookup counter will be increased.

    Parameters
    ----------
    address : str
        The IP Address to look up.
    conn : Union[asyncpg.Connection, asyncpg.pool.Pool]
        The connection to the database.
    session : aiohttp.ClientSession
        The session to fetch address info with if it's not cached.
    token : str
        The API token to look up addresses.
    """

    info = await fetch_cached_info(address, conn=conn)

    if info is not None:
        if (datetime.datetime.now() - info.get("last_updated")) > datetime.timedelta(days=7):
            log.info(f'IP info for address "{address}" old, updating ..')
            info = await update_full_address_info(address, conn=conn, session=session, token=token)
        else:
            log.info(f'IP info for address "{address}" cached.')
    else:
        log.info(f'IP info for address "{address}" not cached, fetching ..')
        info = await fetch_address_info(address, conn=conn, session=session, token=token)
    try:
        return format_response(info)
    except KeyError as KE:
        log.info("KeyError {}".format(KE))
        info = await update_address_info(address, conn=conn, session=session, token=token)
        log.info("Updated loc for {}".format(address))
        return format_response(info)

async def fetch_cached_info(address, *, conn):
    record = await conn.fetchrow(
        'SELECT ip, country_code, accessed, city, region, org, last_updated, loc FROM ip_storage WHERE ip = $1', address
    )

    if record is None:
        return None

    await conn.execute('UPDATE ip_storage SET accessed = ip_storage.accessed + 1 WHERE ip = $1', address)
    return dict(record)

async def update_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        'UPDATE ip_storage SET loc = $2 WHERE ip = $1',
        address,
        loc
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

async def update_full_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        """
        UPDATE ip_storage
        SET country_code = $1,
        city = $2,
        region = $3,
        org = $4,
        loc = $5,
        last_updated = NOW()
        WHERE ip = $6
        """,
        country_code, city, region, org, loc, address
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

async def fetch_address_info(address, *, conn, session, token):
    async with session.get(f'https://www.ipinfo.io/{address}/json?token={token}') as resp:
        data = await resp.json()

    city = data.get('city', '??')
    country_code = data.get('country', '??')
    org = data.get('org', '??')
    region = data.get('region', '??')
    loc = data.get('loc', '??')

    await conn.execute(
        'INSERT INTO ip_storage(ip, country_code, city, region, org, loc) VALUES ($1, $2, $3, $4, $5, $6)',
        address,
        country_code,
        city,
        region,
        org,
        loc
    )

    result = {
        'ip': ipaddress.ip_address(address),
        'country_code': country_code,
        'accessed': 1,
        'city': city,
        'region': region,
        'org': org,
        'loc': loc,
        'last_updated': datetime.datetime.utcnow()
    }

    return result

def format_response(info):
    ip = info['ip']
    country_code = info['country_code']

    response = {
        'global': ip.is_global,
        'loopback': ip.is_loopback,
        'max_prefixlen': ip.max_prefixlen,
        'multicast': ip.is_multicast,
        'link_local': ip.is_link_local,
        'private': ip.is_private,
        'reserved': ip.is_reserved,
        'reverse_pointer': ip.reverse_pointer,
        'unspecified': ip.is_unspecified,
        'version': ip.version,

        'city': info['city'],
        'country': codes.get(country_code, '??'),
        'country_code': country_code,
        'region': info['region'],
        'org': info['org'],
        'loc': info['loc']
    }

    if isinstance(ip, ipaddress.IPv6Address):
        response.update({
            'ipv4_mapped': ip.ipv4_mapped,
            'site_local': ip.is_site_local,
            'sixtofour': ip.sixtofour,
            'teredo': ip.teredo,
        })

    return response

# API statistics

async def fetch_lookup_stats(countries, *, conn):
    """
    Fetch how many times all, a single or multiple countries have been looked up via the API.

    Parameters
    ----------
    countries : Optional[List[str]]
        The countries to look up. If None all countries will be looked up.
    conn : Union[asyncpg.Connection, asyncpg.pool.Pool]
        The connection to the database.
    """

    if countries is not None:
        records = await conn.fetch(
            """
            SELECT count(*) AS stored_ips, CAST (sum(accessed) AS INTEGER) AS lookups, country_code
            FROM ip_storage
            WHERE country_code = ANY($1::TEXT[])
            GROUP BY country_code
            """,
            countries,
        )
    else:
        records = await conn.fetch(
            """
            SELECT count(*) AS stored_ips, CAST (sum(accessed) AS INTEGER) AS lookups, country_code
            FROM ip_storage
            GROUP BY country_code
            """
        )

    results = {
        x['country_code']: {
            'country': codes.get(x['country_code'], '??'), 'lookups': x['lookups'], 'stored_ips': x['stored_ips']
        } for x in records
    }

    return results
<|endoftext|>


<commit_before>#!/usr/local/bin/python

import cPickle
import datetime
import os
import urllib2
import xml.etree.ElementTree as ET
from decimal import *
from urllib import urlencode

# Aurgh
from django.core.management import setup_environ
import settings
setup_environ(settings)

from thing.models import *

BASE_URL = 'http://api.eve-online.com'
CHARACTERS_URL = '%s/account/Characters.xml.aspx' % (BASE_URL)
ORDERS_URL = '%s/corp/MarketOrders.xml.aspx' % (BASE_URL)
TRANSACTIONS_URL = '%s/corp/WalletTransactions.xml.aspx' % (BASE_URL)
WALLET_URL = '%s/corp/AccountBalance.xml.aspx' % (BASE_URL)

PADDING = datetime.timedelta(minutes=1)

def main():
	_now = datetime.datetime.now

	# Load cache
	cache_path = os.path.dirname(os.path.abspath(__file__))
	pickle_filepath = os.path.join(cache_path, 'cache/cache.pickle')
	if os.path.exists(pickle_filepath):
		cache = cPickle.load(open(pickle_filepath, 'rb'))
	else:
		cache = {
			'char': {},
			'corp': {},
		}

	for character in Character.objects.all():
		# Skip if they have no valid user_id/api_key
		if not character.eve_user_id or len(character.eve_api_key) != 64:
			continue

		# Initialise cache
		if character.name not in cache['char']:
			cache['char'][character.name] = {}

		# Get their character ID if it has never been retrieved
		if not character.eve_character_id:
			root, times = fetch_api(CHARACTERS_URL, {}, character)
			err = root.find('error')
			if err is not None:
				show_error('character', err, times)
			else:
				for row in root.findall('result/rowset/row'):
					if row.attrib['name'].lower() == character.name.lower():
						character.eve_character_id = row.attrib['characterID']
						character.save()

		corporation = character.corporation

		# Initialise cache
		if corporation.name not in cache['corp']:
			cache['corp'][corporation.name] = {
				'balances': datetime.datetime(1900, 1, 1),
				'orders': datetime.datetime(1900, 1, 1),
				'transactions': {},
			}

		# Update corporation wallet information/balances
		if _now() > cache['corp'][corporation.name]['balances']:
			root, times = fetch_api(WALLET_URL, {'characterID': character.eve_character_id}, character)
			err = root.find('error')
			if err is not None:
				show_error('corpwallet', err, times)
				#print 'DEBUG: now: %s | bc: %s' % (_now(), cache['corp'][corporation.name]['balances'])
			else:
				for row in root.findall('result/rowset/row'):
					accountID = int(row.attrib['accountID'])
					accountKey = int(row.attrib['accountKey'])
					balance = Decimal(row.attrib['balance'])

					wallet = CorpWallet.objects.filter(pk=accountID)
					if wallet:
						wallet[0].balance = balance
						wallet[0].save()
					else:
						wallet = CorpWallet(account_id=accountID, corporation=corporation, account_key=accountKey, balance=balance)
						wallet.save()

			cache['corp'][corporation.name]['balances'] = _now() + times['delta'] + PADDING

		# Update corporation transactions
		tcache = cache['corp'][corporation.name]['transactions']
		for wallet in CorpWallet.objects.filter(corporation=corporation):
			wak = wallet.account_key
			if wak in tcache and _now() < tcache[wak]:
				continue

			params = {
				'characterID': character.eve_character_id,
				'accountKey': wak,
			}

			one_week_ago = None

			while 1:
				root, times = fetch_api(TRANSACTIONS_URL, params, character)
				err = root.find('error')
				if err is not None:
					# Fuck it, the API flat out lies about cache times
					if err.attrib['code'] not in ('101', '103'):
						show_error('corptrans', err, times)
					break

				# We need to stop asking for data if the oldest transaction entry is older
				# than one week
				if one_week_ago is None:
					one_week_ago = times['current'] - datetime.timedelta(7)

				rows = root.findall('result/rowset/row')
				if not rows:
					break

				for row in rows:
					transaction_time = parse_api_date(row.attrib['transactionDateTime'])

					# Skip already seen transactions
					transaction_id = int(row.attrib['transactionID'])
					if Transaction.objects.filter(pk=transaction_id):
						continue

					# Make sure the item typeID is valid
					items = Item.objects.filter(pk=row.attrib['typeID'])
					if items.count() == 0:
						print "ERROR: item with typeID '%s' does not exist, what the fuck?" % (row.attrib['typeID'])
						print '>> attrib = %r' % (row.attrib)
						continue

					# Make the station object if it doesn't already exist
					station = get_station(int(row.attrib['stationID']), row.attrib['stationName'])

					# Make the transaction object
					quantity = int(row.attrib['quantity'])
					price = Decimal(row.attrib['price'])
					t = Transaction(
						id=transaction_id,
						corporation=corporation,
						corp_wallet=wallet,
						date=transaction_time,
						t_type=row.attrib['transactionType'][0].upper(),
						station=station,
						item=items[0],
						quantity=quantity,
						price=price,
						total_price=quantity * price,
					)
					t.save()

					#print t.id, t.date, t.t_type, t.item, t.quantity, t.price

				# If we got 1000 rows we should retrieve some more
				#print 'DEBUG: rows: %d | cur: %s | owa: %s | tt: %s' % (len(rows), times['current'], one_week_ago, transaction_time)
				if len(rows) == 1000 and transaction_time > one_week_ago:
					params['beforeTransID'] = transaction_id
				else:
					break

			#print 'DEBUG: tcw: %s' % (tcache.get(wak, 0))
			tcache[wak] = _now() + times['delta'] + PADDING
			#print 'DEBUG: wak: %s | now: %s | tcw: %s' % (wak, _now(), tcache[wak])

		# Update corporation orders
		if _now() > cache['corp'][corporation.name]['orders']:
			params = {
				'characterID': character.eve_character_id,
			}

			root, times = fetch_api(ORDERS_URL, params, character)
			err = root.find('error')
			if err is not None:
				show_error('corporders', err, times)

			else:
				rows = root.findall('result/rowset/row')
				if not rows:
					break

				for row in rows:
					order_id = int(row.attrib['orderID'])
					orders = Order.objects.filter(pk=order_id)

					# Hey, this transaction already exists
					if orders:
						order = orders[0]

						# Order is active, update stuff I guess
						if row.attrib['orderState'] == '0':
							order.issued = parse_api_date(row.attrib['issued'])
							order.volume_remaining = int(row.attrib['volRemaining'])
							order.escrow = Decimal(row.attrib['escrow'])
							order.price = Decimal(row.attrib['price'])
							order.total_price = order.volume_remaining * order.price
							order.save()

						# Not active, nuke it from orbit
						else:
							order.delete()

					# Doesn't exist and is active, make a new order
					elif row.attrib['orderState'] == '0':
						if row.attrib['bid'] == '0':
							o_type = 'S'
						else:
							o_type = 'B'

						# Make sure the character charID is valid
						chars = Character.objects.filter(eve_character_id=row.attrib['charID'])
						if not chars:
							print 'ERROR: no matching Character object for charID=%s' % (row.attrib['charID'])
							continue

						# Make sure the item typeID is valid
						items = Item.objects.filter(pk=row.attrib['typeID'])
						if items.count() == 0:
							print "ERROR: item with typeID '%s' does not exist, what the fuck?" % (row.attrib['typeID'])
							print '>> attrib = %r' % (row.attrib)
							continue

						remaining = int(row.attrib['volRemaining'])
						price = Decimal(row.attrib['price'])
						order = Order(
							id=order_id,
							corporation=corporation,
							corp_wallet=CorpWallet.objects.filter(corporation=corporation, account_key=row.attrib['accountKey'])[0],
							character=chars[0],
							station=get_station(int(row.attrib['stationID']), 'UNKNOWN STATION'),
							item=items[0],
							issued=parse_api_date(row.attrib['issued']),
							o_type=o_type,
							volume_entered=int(row.attrib['volEntered']),
							volume_remaining=remaining,
							min_volume=int(row.attrib['minVolume']),
							duration=int(row.attrib['duration']),
							escrow=Decimal(row.attrib['escrow']),
							price=price,
							total_price=remaining*price,
						)
						order.save()

			cache['corp'][corporation.name]['orders'] = _now() + times['delta'] + PADDING

	# Save cache
	cPickle.dump(cache, open(pickle_filepath, 'wb'))

def fetch_api(url, params, character):
	#print 'params: %s' % (params)
	params['userID'] = character.eve_user_id
	params['apiKey'] = character.eve_api_key

	f = urllib2.urlopen(url, urlencode(params))
	data = f.read()
	f.close()

	root = ET.fromstring(data)
	times = {
		'current': parse_api_date(root.find('currentTime').text),
		'until': parse_api_date(root.find('cachedUntil').text),
	}
	times['delta'] = times['until'] - times['current']

	#print '%s | currentTime: %s | cachedUntil: %s | delta: %s' % (url, times['current'], times['until'], times['delta'])

	return (root, times)

def parse_api_date(s):
	return datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S')

def show_error(text, err, times):
	print '(%s) %s: %s | %s -> %s' % (text, err.attrib['code'], err.text, times['current'], times['until'])

def get_station(station_id, station_name):
	station = Station.objects.filter(pk=station_id)
	if station:
		station = station[0]
	else:
		station = Station(id=station_id, name=station_name)
		station.save()
	return station

if __name__ == '__main__':
	main()
<commit_msg>Minor bug fix.<commit_after>#!/usr/local/bin/python

import cPickle
import datetime
import os
import urllib2
import xml.etree.ElementTree as ET
from decimal import *
from urllib import urlencode

# Aurgh
from django.core.management import setup_environ
import settings
setup_environ(settings)

from thing.models import *

BASE_URL = 'http://api.eve-online.com'
CHARACTERS_URL = '%s/account/Characters.xml.aspx' % (BASE_URL)
ORDERS_URL = '%s/corp/MarketOrders.xml.aspx' % (BASE_URL)
TRANSACTIONS_URL = '%s/corp/WalletTransactions.xml.aspx' % (BASE_URL)
WALLET_URL = '%s/corp/AccountBalance.xml.aspx' % (BASE_URL)

PADDING = datetime.timedelta(minutes=1)

def main():
	_now = datetime.datetime.now

	# Load cache
	cache_path = os.path.dirname(os.path.abspath(__file__))
	pickle_filepath = os.path.join(cache_path, 'cache/cache.pickle')
	if os.path.exists(pickle_filepath):
		cache = cPickle.load(open(pickle_filepath, 'rb'))
	else:
		cache = {
			'char': {},
			'corp': {},
		}

	for character in Character.objects.all():
		# Skip if they have no valid user_id/api_key
		if not character.eve_user_id or len(character.eve_api_key) != 64:
			continue

		# Initialise cache
		if character.name not in cache['char']:
			cache['char'][character.name] = {}

		# Get their character ID if it has never been retrieved
		if not character.eve_character_id:
			root, times = fetch_api(CHARACTERS_URL, {}, character)
			err = root.find('error')
			if err is not None:
				show_error('character', err, times)
			else:
				for row in root.findall('result/rowset/row'):
					if row.attrib['name'].lower() == character.name.lower():
						character.eve_character_id = row.attrib['characterID']
						character.save()

		corporation = character.corporation

		# Initialise cache
		if corporation.name not in cache['corp']:
			cache['corp'][corporation.name] = {
				'balances': datetime.datetime(1900, 1, 1),
				'orders': datetime.datetime(1900, 1, 1),
				'transactions': {},
			}

		# Update corporation wallet information/balances
		if _now() > cache['corp'][corporation.name]['balances']:
			root, times = fetch_api(WALLET_URL, {'characterID': character.eve_character_id}, character)
			err = root.find('error')
			if err is not None:
				show_error('corpwallet', err, times)
				#print 'DEBUG: now: %s | bc: %s' % (_now(), cache['corp'][corporation.name]['balances'])
			else:
				for row in root.findall('result/rowset/row'):
					accountID = int(row.attrib['accountID'])
					accountKey = int(row.attrib['accountKey'])
					balance = Decimal(row.attrib['balance'])

					wallet = CorpWallet.objects.filter(pk=accountID)
					if wallet:
						wallet[0].balance = balance
						wallet[0].save()
					else:
						wallet = CorpWallet(account_id=accountID, corporation=corporation, account_key=accountKey, balance=balance)
						wallet.save()

			cache['corp'][corporation.name]['balances'] = _now() + times['delta'] + PADDING

		# Update corporation transactions
		tcache = cache['corp'][corporation.name]['transactions']
		for wallet in CorpWallet.objects.filter(corporation=corporation):
			wak = wallet.account_key
			if wak in tcache and _now() < tcache[wak]:
				continue

			params = {
				'characterID': character.eve_character_id,
				'accountKey': wak,
			}

			one_week_ago = None

			while 1:
				root, times = fetch_api(TRANSACTIONS_URL, params, character)
				err = root.find('error')
				if err is not None:
					# Fuck it, the API flat out lies about cache times
					if err.attrib['code'] not in ('101', '103'):
						show_error('corptrans', err, times)
					break

				# We need to stop asking for data if the oldest transaction entry is older
				# than one week
				if one_week_ago is None:
					one_week_ago = times['current'] - datetime.timedelta(7)

				rows = root.findall('result/rowset/row')
				if not rows:
					break

				for row in rows:
					transaction_time = parse_api_date(row.attrib['transactionDateTime'])

					# Skip already seen transactions
					transaction_id = int(row.attrib['transactionID'])
					if Transaction.objects.filter(pk=transaction_id).count() > 0:
						continue

					# Make sure the item typeID is valid
					items = Item.objects.filter(pk=row.attrib['typeID'])
					if items.count() == 0:
						print "ERROR: item with typeID '%s' does not exist, what the fuck?" % (row.attrib['typeID'])
						print '>> attrib = %r' % (row.attrib)
						continue

					# Make the station object if it doesn't already exist
					station = get_station(int(row.attrib['stationID']), row.attrib['stationName'])

					# Make the transaction object
					quantity = int(row.attrib['quantity'])
					price = Decimal(row.attrib['price'])
					t = Transaction(
						id=transaction_id,
						corporation=corporation,
						corp_wallet=wallet,
						date=transaction_time,
						t_type=row.attrib['transactionType'][0].upper(),
						station=station,
						item=items[0],
						quantity=quantity,
						price=price,
						total_price=quantity * price,
					)
					t.save()

					#print t.id, t.date, t.t_type, t.item, t.quantity, t.price

				# If we got 1000 rows we should retrieve some more
				#print 'DEBUG: rows: %d | cur: %s | owa: %s | tt: %s' % (len(rows), times['current'], one_week_ago, transaction_time)
				if len(rows) == 1000 and transaction_time > one_week_ago:
					params['beforeTransID'] = transaction_id
				else:
					break

			#print 'DEBUG: tcw: %s' % (tcache.get(wak, 0))
			tcache[wak] = _now() + times['delta'] + PADDING
			#print 'DEBUG: wak: %s | now: %s | tcw: %s' % (wak, _now(), tcache[wak])

		# Update corporation orders
		if _now() > cache['corp'][corporation.name]['orders']:
			params = {
				'characterID': character.eve_character_id,
			}

			root, times = fetch_api(ORDERS_URL, params, character)
			err = root.find('error')
			if err is not None:
				show_error('corporders', err, times)

			else:
				rows = root.findall('result/rowset/row')
				if not rows:
					break

				for row in rows:
					order_id = int(row.attrib['orderID'])
					orders = Order.objects.filter(pk=order_id)

					# Hey, this transaction already exists
					if orders:
						order = orders[0]

						# Order is active, update stuff I guess
						if row.attrib['orderState'] == '0':
							order.issued = parse_api_date(row.attrib['issued'])
							order.volume_remaining = int(row.attrib['volRemaining'])
							order.escrow = Decimal(row.attrib['escrow'])
							order.price = Decimal(row.attrib['price'])
							order.total_price = order.volume_remaining * order.price
							order.save()

						# Not active, nuke it from orbit
						else:
							order.delete()

					# Doesn't exist and is active, make a new order
					elif row.attrib['orderState'] == '0':
						if row.attrib['bid'] == '0':
							o_type = 'S'
						else:
							o_type = 'B'

						# Make sure the character charID is valid
						chars = Character.objects.filter(eve_character_id=row.attrib['charID'])
						if not chars:
							print 'ERROR: no matching Character object for charID=%s' % (row.attrib['charID'])
							continue

						# Make sure the item typeID is valid
						items = Item.objects.filter(pk=row.attrib['typeID'])
						if items.count() == 0:
							print "ERROR: item with typeID '%s' does not exist, what the fuck?" % (row.attrib['typeID'])
							print '>> attrib = %r' % (row.attrib)
							continue

						remaining = int(row.attrib['volRemaining'])
						price = Decimal(row.attrib['price'])
						order = Order(
							id=order_id,
							corporation=corporation,
							corp_wallet=CorpWallet.objects.filter(corporation=corporation, account_key=row.attrib['accountKey'])[0],
							character=chars[0],
							station=get_station(int(row.attrib['stationID']), 'UNKNOWN STATION'),
							item=items[0],
							issued=parse_api_date(row.attrib['issued']),
							o_type=o_type,
							volume_entered=int(row.attrib['volEntered']),
							volume_remaining=remaining,
							min_volume=int(row.attrib['minVolume']),
							duration=int(row.attrib['duration']),
							escrow=Decimal(row.attrib['escrow']),
							price=price,
							total_price=remaining*price,
						)
						order.save()

			cache['corp'][corporation.name]['orders'] = _now() + times['delta'] + PADDING

	# Save cache
	cPickle.dump(cache, open(pickle_filepath, 'wb'))

def fetch_api(url, params, character):
	#print 'params: %s' % (params)
	params['userID'] = character.eve_user_id
	params['apiKey'] = character.eve_api_key

	f = urllib2.urlopen(url, urlencode(params))
	data = f.read()
	f.close()

	root = ET.fromstring(data)
	times = {
		'current': parse_api_date(root.find('currentTime').text),
		'until': parse_api_date(root.find('cachedUntil').text),
	}
	times['delta'] = times['until'] - times['current']

	#print '%s | currentTime: %s | cachedUntil: %s | delta: %s' % (url, times['current'], times['until'], times['delta'])

	return (root, times)

def parse_api_date(s):
	return datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S')

def show_error(text, err, times):
	print '(%s) %s: %s | %s -> %s' % (text, err.attrib['code'], err.text, times['current'], times['until'])

def get_station(station_id, station_name):
	station = Station.objects.filter(pk=station_id)
	if station:
		station = station[0]
	else:
		station = Station(id=station_id, name=station_name)
		station.save()
	return station

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    longboxed.manage.comics
    ~~~~~~~~~~~~~~~~~~~~~~~

    comic management commands
"""
import csv
from StringIO import StringIO
from flask import current_app
from flask.ext.script import Command, Option, prompt, prompt_bool

from ..helpers import mail_content
from ..importer import NewDailyDownloadImporter, NewWeeklyReleasesImporter, WeeklyReleasesImporter, WeeklyReleaseRecord
from ..services import comics
from ..models import Issue

class TestCommand(Command):
    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
        ]

    def run(self, week):
        fieldnames = [x[2] for x in current_app.config['RELEASE_CSV_RULES']]
        issue_releaser = NewWeeklyReleasesImporter()
        issue_releaser.run(
            csv_fieldnames = fieldnames,
            supported_publishers = current_app.config['SUPPORTED_DIAMOND_PUBS'],
            affiliate_id = current_app.config['AFFILIATE_ID'],
            week = week
        )
        return

class ScheduleReleasesCommand(Command):
    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
        ]

    def run(self, week):
        release_instance = WeeklyReleasesImporter(
            week=week,
            affiliate_id=current_app.config['AFFILIATE_ID'],
            supported_publishers=current_app.config['SUPPORTED_DIAMOND_PUBS'],
            csv_rules=current_app.config['RELEASE_CSV_RULES'],
            record=WeeklyReleaseRecord
        )
        scheduled_releases = release_instance.run()
        return

# class ImportDatabase(Command):
#     def get_options(self):
#         return [
#             Option('--days', '-d', dest='days', default=21, type=int)
#         ]

#     def run(self, days):
#         print 'Importing the next %d days worth of comic books...' % days
#         import_instance = DailyDownloadImporter(
#             days=days,
#             affiliate_id=current_app.config['AFFILIATE_ID'],
#             supported_publishers=current_app.config['SUPPORTED_PUBS'],
#             csv_rules=current_app.config['CSV_RULES'],
#             record=DailyDownloadRecord
#         )
#         imported_issues = import_instance.run()
#         return

class ImportDatabase(Command):
    def get_options(self):
        return [
            Option('--days', '-d', dest='days', default=21, type=int)
        ]

    def run(self, days):
        fieldnames = [x[2] for x in current_app.config['CSV_RULES']]
        database_importer = NewDailyDownloadImporter()
        database_importer.run(
            csv_fieldnames = fieldnames,
            supported_publishers = current_app.config['SUPPORTED_PUBS'],
            affiliate_id = current_app.config['AFFILIATE_ID'],
            thumbnail_widths = current_app.config['THUMBNAIL_WIDTHS'],
            days = days
        )
        return

class SetCoverImageCommand(Command):
    """
    Sets the cover image of an issue. The issues is found in the
    database with a diamond id. If the issue already has an image
    attached, you can optionally choose to replace it.
    """
    def run(self):
        diamond_id = prompt('Issue Diamond id')
        issue = Issue.query.filter_by(diamond_id=diamond_id).first()
        if issue:
            url = prompt('Url of jpg image for cover image')
            overwrite = False
            if issue.cover_image.original:
                print 'Issue object already has a cover image set.'
                overwrite = prompt_bool('Overwrite existing picture?')
            success = issue.set_cover_image_from_url(url, overwrite=overwrite)
            if success:
                print 'Successfully set cover image'
            return
        print 'No issue found!'
        return

class CrossCheckCommand(Command):
    """
    Cross checks releases with items in database and mails missing
    issues to a provided email address
    """

    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
            Option('-e', '--email', dest='email', default='timbueno@gmail.com')
        ]

    def run(self, email, week):
        if week == 'twoweeks':
            raise NotImplementedError
        content = comics.get_shipping_from_TFAW(week)
        shipping = comics.get_issue_dict_shipping(content)
        not_in_db = [i for i in shipping if not comics.issues.first(diamond_id=i['ITEMCODE'])]
        not_in_db = sorted(not_in_db, key=lambda x: x['Vendor'])
        f = StringIO()
        ordered_fieldnames = ['ITEMCODE', 'DiscountCode', 'TITLE', 'Vendor', 'PRICE']
        outcsv = csv.DictWriter(f, fieldnames=ordered_fieldnames, delimiter='\t')
        for i in not_in_db:
            outcsv.writerow(i)
        mail_content([email], 'checker@longboxed.com', 'Testing', 'Attached is your checks', f.getvalue())
        f.close()
        return

# class UpdateDatabaseCommand(Command):
#     """Updates database with TFAW Daily Download"""

#     def get_options(self):
#         return [
#             Option('-d','--days', dest='days', required=True)
#         ]

#     def run(self, days):
#         print 'Starting update'
#         days = int(days)
#         comics.add_new_issues_to_database(days=days)
#         print 'Done Adding to DB'<commit_msg>removed old release command - replaced with new one.<commit_after># -*- coding: utf-8 -*-
"""
    longboxed.manage.comics
    ~~~~~~~~~~~~~~~~~~~~~~~

    comic management commands
"""
import csv
from StringIO import StringIO
from flask import current_app
from flask.ext.script import Command, Option, prompt, prompt_bool

from ..helpers import mail_content
from ..importer import NewDailyDownloadImporter, NewWeeklyReleasesImporter
from ..services import comics
from ..models import Issue

class TestCommand(Command):
    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
        ]

    def run(self, week):
        fieldnames = [x[2] for x in current_app.config['RELEASE_CSV_RULES']]
        issue_releaser = NewWeeklyReleasesImporter()
        issue_releaser.run(
            csv_fieldnames = fieldnames,
            supported_publishers = current_app.config['SUPPORTED_DIAMOND_PUBS'],
            affiliate_id = current_app.config['AFFILIATE_ID'],
            week = week
        )
        return

class ScheduleReleasesCommand(Command):
    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
        ]

    def run(self, week):
        fieldnames = [x[2] for x in current_app.config['RELEASE_CSV_RULES']]
        issue_releaser = NewWeeklyReleasesImporter()
        issue_releaser.run(
            csv_fieldnames = fieldnames,
            supported_publishers = current_app.config['SUPPORTED_DIAMOND_PUBS'],
            affiliate_id = current_app.config['AFFILIATE_ID'],
            week = week
        )
        return

# class ScheduleReleasesCommand(Command):
#     def get_options(self):
#         return [
#             Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
#         ]

#     def run(self, week):
#         release_instance = WeeklyReleasesImporter(
#             week=week,
#             affiliate_id=current_app.config['AFFILIATE_ID'],
#             supported_publishers=current_app.config['SUPPORTED_DIAMOND_PUBS'],
#             csv_rules=current_app.config['RELEASE_CSV_RULES'],
#             record=WeeklyReleaseRecord
#         )
#         scheduled_releases = release_instance.run()
#         return

# class ImportDatabase(Command):
#     def get_options(self):
#         return [
#             Option('--days', '-d', dest='days', default=21, type=int)
#         ]

#     def run(self, days):
#         print 'Importing the next %d days worth of comic books...' % days
#         import_instance = DailyDownloadImporter(
#             days=days,
#             affiliate_id=current_app.config['AFFILIATE_ID'],
#             supported_publishers=current_app.config['SUPPORTED_PUBS'],
#             csv_rules=current_app.config['CSV_RULES'],
#             record=DailyDownloadRecord
#         )
#         imported_issues = import_instance.run()
#         return

class ImportDatabase(Command):
    def get_options(self):
        return [
            Option('--days', '-d', dest='days', default=21, type=int)
        ]

    def run(self, days):
        fieldnames = [x[2] for x in current_app.config['CSV_RULES']]
        database_importer = NewDailyDownloadImporter()
        database_importer.run(
            csv_fieldnames = fieldnames,
            supported_publishers = current_app.config['SUPPORTED_PUBS'],
            affiliate_id = current_app.config['AFFILIATE_ID'],
            thumbnail_widths = current_app.config['THUMBNAIL_WIDTHS'],
            days = days
        )
        return

class SetCoverImageCommand(Command):
    """
    Sets the cover image of an issue. The issues is found in the
    database with a diamond id. If the issue already has an image
    attached, you can optionally choose to replace it.
    """
    def run(self):
        diamond_id = prompt('Issue Diamond id')
        issue = Issue.query.filter_by(diamond_id=diamond_id).first()
        if issue:
            url = prompt('Url of jpg image for cover image')
            overwrite = False
            if issue.cover_image.original:
                print 'Issue object already has a cover image set.'
                overwrite = prompt_bool('Overwrite existing picture?')
            success = issue.set_cover_image_from_url(url, overwrite=overwrite)
            if success:
                print 'Successfully set cover image'
            return
        print 'No issue found!'
        return

class CrossCheckCommand(Command):
    """
    Cross checks releases with items in database and mails missing
    issues to a provided email address
    """

    def get_options(self):
        return [
            Option('-w', '--week', dest='week', required=True, choices=['thisweek', 'nextweek', 'twoweeks']),
            Option('-e', '--email', dest='email', default='timbueno@gmail.com')
        ]

    def run(self, email, week):
        if week == 'twoweeks':
            raise NotImplementedError
        content = comics.get_shipping_from_TFAW(week)
        shipping = comics.get_issue_dict_shipping(content)
        not_in_db = [i for i in shipping if not comics.issues.first(diamond_id=i['ITEMCODE'])]
        not_in_db = sorted(not_in_db, key=lambda x: x['Vendor'])
        f = StringIO()
        ordered_fieldnames = ['ITEMCODE', 'DiscountCode', 'TITLE', 'Vendor', 'PRICE']
        outcsv = csv.DictWriter(f, fieldnames=ordered_fieldnames, delimiter='\t')
        for i in not_in_db:
            outcsv.writerow(i)
        mail_content([email], 'checker@longboxed.com', 'Testing', 'Attached is your checks', f.getvalue())
        f.close()
        return

# class UpdateDatabaseCommand(Command):
#     """Updates database with TFAW Daily Download"""

#     def get_options(self):
#         return [
#             Option('-d','--days', dest='days', required=True)
#         ]

#     def run(self, days):
#         print 'Starting update'
#         days = int(days)
#         comics.add_new_issues_to_database(days=days)
#         print 'Done Adding to DB'<|endoftext|>


<commit_before>import os
from os.path import dirname, abspath

from celery.schedules import crontab

def media_directory(path):
    """Return absolute path to subdirectory of MEDIA_ROOT.
    The directory will be created if it doesn't exist.
    """
    abs_path = os.path.join(MEDIA_ROOT, path)
    if not os.path.isdir(abs_path):
        os.makedirs(abs_path)
    return abs_path

CLIENT_VERSION = "0.4.23"

DEBUG = False
TEMPLATE_DEBUG = DEBUG
THUMBNAIL_DEBUG = False

BASE_DIR = dirname(dirname(dirname(abspath(__file__))))

ADMINS = (
    ('Mathieu Comandon', 'strycore@gmail.com'),
)
MANAGERS = ADMINS
INTERNAL_IPS = ('127.0.0.1',)
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'lutris.db'),
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        'LOCATION': '127.0.0.1:11211',
    }
}

TIME_ZONE = 'Etc/UTC'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
STATICFILES_DIRS = (
    os.path.join(BASE_DIR, "public"),
    os.path.join(BASE_DIR, "frontend/vue/dist"),
    os.path.join(BASE_DIR, "components"),
)
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# Media files
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

FILES_ROOT = media_directory('files')
FILES_URL = 'http://localhost:8000/media/files/'

TOSEC_PATH = media_directory('tosec')
TOSEC_DAT_PATH = os.path.join(TOSEC_PATH)

# TheGamesDB directories
TGD_ROOT = media_directory('thegamesdb')
TGD_CLEAR_LOGO_PATH = media_directory('thegamesdb/clearlogo')
TGD_BANNER_PATH = media_directory('thegamesdb/banners')
TGD_SCREENSHOT_PATH = media_directory('thegamesdb/screenshot')
TGD_FANART_PATH = media_directory('thegamesdb/fanart')
TGD_LUTRIS_BANNER_PATH = media_directory('thegamesdb/lutris-banners')

SECRET_KEY = os.environ.get('SECRET_KEY', 'changeme')

MIDDLEWARE = [
    # Caching disabled until proper invalidation is implemented
    # 'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    # 'django.middleware.cache.FetchFromCacheMiddleware',
]

ROOT_URLCONF = 'lutrisweb.urls'
WSGI_APPLICATION = 'lutrisweb.wsgi.application'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, "templates"),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                "django.contrib.auth.context_processors.auth",
                "django.template.context_processors.debug",
                "django.template.context_processors.i18n",
                "django.template.context_processors.media",
                "django.template.context_processors.static",
                "django.template.context_processors.tz",
                "django.template.context_processors.request",
                "django.contrib.messages.context_processors.messages",
                "common.context_processors.discord_url",
            ],
            'debug': False
        }
    }
]

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.humanize',
    'grappelli',
    'django.contrib.admin',
    'django.contrib.admindocs',

    'sorl.thumbnail',
    'rest_framework',
    'rest_framework.authtoken',
    'rest_framework_swagger',
    'croppie',
    'crispy_forms',
    'django_select2',
    'markupfield',
    'django_openid_auth',
    'django_extensions',
    'reversion',

    'common',
    'platforms',
    'games',
    'bundles',
    'runners',
    'accounts',
    'tosec',
]

BANNER_SIZE = "184x69"
ICON_SIZE = "32x32"
ICON_LARGE_SIZE = "256x256"
THUMBNAIL_ENGINE = 'sorl.thumbnail.engines.convert_engine.Engine'
THUMBNAIL_COLORSPACE = "sRGB"

AUTH_USER_MODEL = 'accounts.User'
AUTH_PROFILE_MODULE = "accounts.Profile"
ACCOUNT_ACTIVATION_DAYS = 3
LOGIN_REDIRECT_URL = "/"
LOGIN_URL = "/user/login/"
AUTHENTICATION_BACKENDS = (
    'django_openid_auth.auth.OpenIDBackend',
    'accounts.backends.SmarterModelBackend',
)
OPENID_SSO_SERVER_URL = 'http://steamcommunity.com/openid'

DISCOURSE_SSO_SECRET = os.environ.get('DISCOURSE_SSO_SECRET')
DISCOURSE_URL = 'https://forums.lutris.net'

DISCORD_URL = "https://discord.gg/C3uJjRD"
DISCORD_ISSUE_WEBHOOK_ID = os.environ.get('DISCORD_ISSUE_WEBHOOK_ID')
DISCORD_ISSUE_WEBHOOK_TOKEN = os.environ.get('DISCORD_ISSUE_WEBHOOK_TOKEN')
# Modify temporarily the session serializer because the json serializer in
# Django 1.6 can't serialize openid.yadis.manager.YadisServiceManager objects
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'

# Crispy forms
CRISPY_TEMPLATE_PACK = 'bootstrap3'
CRISPY_FAIL_SILENTLY = not DEBUG

# Admin
GRAPPELLI_ADMIN_TITLE = "Lutris Administration"

# Select2
SELECT2_JS = '/static/js/select2.min.js'
SELECT2_CSS = '/static/css/select2.min.css'

# Email

try:
    SEND_EMAILS = bool(int(os.environ.get('SEND_EMAILS', '1')))
except ValueError:
    SEND_EMAILS = True
DEFAULT_FROM_EMAIL = "admin@lutris.net"
SERVER_EMAIL = "admin@lutris.net"
EMAIL_SUBJECT_PREFIX = "[Lutris] "

# Celery
CELERY_SEND_TASK_ERROR_EMAILS = True

CELERYBEAT_SCHEDULE = {
    'send-daily-mod-mail': {
        'task': 'accounts.tasks.daily_mod_mail',
        'schedule': crontab(hour=18, minute=0),
    },
    'delete-unchanged-forks': {
        'task': 'games.tasks.delete_unchanged_forks',
        'schedule': crontab(hour=17, minute=59)
    }
}
BROKER_URL = 'amqp://guest:guest@localhost//'
# API Keys
STEAM_API_KEY = "********************************"

# Rest Framework
REST_FRAMEWORK = {
    'DEFAULT_THROTTLE_CLASSES': (
        'rest_framework.throttling.AnonRateThrottle',
        'rest_framework.throttling.UserRateThrottle'
    ),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.TokenAuthentication',
        'rest_framework.authentication.SessionAuthentication',
    ),
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 250,
}

# Shell Plus
SHELL_PLUS_DONT_LOAD = ['tosec']

SILENCED_SYSTEM_CHECKS = [
    'urls.W002',
]

# Logging
SEND_BROKEN_LINK_EMAILS = False
LOGGING_HANDLERS = ['file', 'mail_admins', 'console']
DEFAULT_LOGGING_CONFIG = {
    'handlers': LOGGING_HANDLERS,
    'level': 'DEBUG',
    'propagate': True,
}
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s '
                      '%(process)d %(thread)d %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        }
    },
    'handlers': {
        'null': {
            'level': 'DEBUG',
            'class': 'logging.NullHandler',
        },
        'mail_admins': {
            'level': 'ERROR',
            'include_html': True,
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'formatter': 'verbose',
            'filename': 'lutrisweb.log'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': LOGGING_HANDLERS,
            'propagate': True,
            'level': 'INFO',
        },
        'factory': {
            'handlers': ['null'],
            'propagate': False,
            'level': 'INFO',
        },
        'django.request': {
            'handlers': LOGGING_HANDLERS,
            'level': 'WARNING',
            'propagate': True,
        },
        'lutrisweb': DEFAULT_LOGGING_CONFIG,
        'accounts': DEFAULT_LOGGING_CONFIG,
        'common': DEFAULT_LOGGING_CONFIG,
        'games': DEFAULT_LOGGING_CONFIG,
        'thegamesdb': DEFAULT_LOGGING_CONFIG
    }
}
<commit_msg>Add log handler for scripts<commit_after>import os
from os.path import dirname, abspath

from celery.schedules import crontab

def media_directory(path):
    """Return absolute path to subdirectory of MEDIA_ROOT.
    The directory will be created if it doesn't exist.
    """
    abs_path = os.path.join(MEDIA_ROOT, path)
    if not os.path.isdir(abs_path):
        os.makedirs(abs_path)
    return abs_path

CLIENT_VERSION = "0.4.23"

DEBUG = False
TEMPLATE_DEBUG = DEBUG
THUMBNAIL_DEBUG = False

BASE_DIR = dirname(dirname(dirname(abspath(__file__))))

ADMINS = (
    ('Mathieu Comandon', 'strycore@gmail.com'),
)
MANAGERS = ADMINS
INTERNAL_IPS = ('127.0.0.1',)
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'lutris.db'),
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        'LOCATION': '127.0.0.1:11211',
    }
}

TIME_ZONE = 'Etc/UTC'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
STATICFILES_DIRS = (
    os.path.join(BASE_DIR, "public"),
    os.path.join(BASE_DIR, "frontend/vue/dist"),
    os.path.join(BASE_DIR, "components"),
)
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# Media files
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

FILES_ROOT = media_directory('files')
FILES_URL = 'http://localhost:8000/media/files/'

TOSEC_PATH = media_directory('tosec')
TOSEC_DAT_PATH = os.path.join(TOSEC_PATH)

# TheGamesDB directories
TGD_ROOT = media_directory('thegamesdb')
TGD_CLEAR_LOGO_PATH = media_directory('thegamesdb/clearlogo')
TGD_BANNER_PATH = media_directory('thegamesdb/banners')
TGD_SCREENSHOT_PATH = media_directory('thegamesdb/screenshot')
TGD_FANART_PATH = media_directory('thegamesdb/fanart')
TGD_LUTRIS_BANNER_PATH = media_directory('thegamesdb/lutris-banners')

SECRET_KEY = os.environ.get('SECRET_KEY', 'changeme')

MIDDLEWARE = [
    # Caching disabled until proper invalidation is implemented
    # 'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    # 'django.middleware.cache.FetchFromCacheMiddleware',
]

ROOT_URLCONF = 'lutrisweb.urls'
WSGI_APPLICATION = 'lutrisweb.wsgi.application'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, "templates"),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                "django.contrib.auth.context_processors.auth",
                "django.template.context_processors.debug",
                "django.template.context_processors.i18n",
                "django.template.context_processors.media",
                "django.template.context_processors.static",
                "django.template.context_processors.tz",
                "django.template.context_processors.request",
                "django.contrib.messages.context_processors.messages",
                "common.context_processors.discord_url",
            ],
            'debug': False
        }
    }
]

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.humanize',
    'grappelli',
    'django.contrib.admin',
    'django.contrib.admindocs',

    'sorl.thumbnail',
    'rest_framework',
    'rest_framework.authtoken',
    'rest_framework_swagger',
    'croppie',
    'crispy_forms',
    'django_select2',
    'markupfield',
    'django_openid_auth',
    'django_extensions',
    'reversion',

    'common',
    'platforms',
    'games',
    'bundles',
    'runners',
    'accounts',
    'tosec',
]

BANNER_SIZE = "184x69"
ICON_SIZE = "32x32"
ICON_LARGE_SIZE = "256x256"
THUMBNAIL_ENGINE = 'sorl.thumbnail.engines.convert_engine.Engine'
THUMBNAIL_COLORSPACE = "sRGB"

AUTH_USER_MODEL = 'accounts.User'
AUTH_PROFILE_MODULE = "accounts.Profile"
ACCOUNT_ACTIVATION_DAYS = 3
LOGIN_REDIRECT_URL = "/"
LOGIN_URL = "/user/login/"
AUTHENTICATION_BACKENDS = (
    'django_openid_auth.auth.OpenIDBackend',
    'accounts.backends.SmarterModelBackend',
)
OPENID_SSO_SERVER_URL = 'http://steamcommunity.com/openid'

DISCOURSE_SSO_SECRET = os.environ.get('DISCOURSE_SSO_SECRET')
DISCOURSE_URL = 'https://forums.lutris.net'

DISCORD_URL = "https://discord.gg/C3uJjRD"
DISCORD_ISSUE_WEBHOOK_ID = os.environ.get('DISCORD_ISSUE_WEBHOOK_ID')
DISCORD_ISSUE_WEBHOOK_TOKEN = os.environ.get('DISCORD_ISSUE_WEBHOOK_TOKEN')
# Modify temporarily the session serializer because the json serializer in
# Django 1.6 can't serialize openid.yadis.manager.YadisServiceManager objects
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'

# Crispy forms
CRISPY_TEMPLATE_PACK = 'bootstrap3'
CRISPY_FAIL_SILENTLY = not DEBUG

# Admin
GRAPPELLI_ADMIN_TITLE = "Lutris Administration"

# Select2
SELECT2_JS = '/static/js/select2.min.js'
SELECT2_CSS = '/static/css/select2.min.css'

# Email

try:
    SEND_EMAILS = bool(int(os.environ.get('SEND_EMAILS', '1')))
except ValueError:
    SEND_EMAILS = True
DEFAULT_FROM_EMAIL = "admin@lutris.net"
SERVER_EMAIL = "admin@lutris.net"
EMAIL_SUBJECT_PREFIX = "[Lutris] "

# Celery
CELERY_SEND_TASK_ERROR_EMAILS = True

CELERYBEAT_SCHEDULE = {
    'send-daily-mod-mail': {
        'task': 'accounts.tasks.daily_mod_mail',
        'schedule': crontab(hour=18, minute=0),
    },
    'delete-unchanged-forks': {
        'task': 'games.tasks.delete_unchanged_forks',
        'schedule': crontab(hour=17, minute=59)
    }
}
BROKER_URL = 'amqp://guest:guest@localhost//'
# API Keys
STEAM_API_KEY = "********************************"

# Rest Framework
REST_FRAMEWORK = {
    'DEFAULT_THROTTLE_CLASSES': (
        'rest_framework.throttling.AnonRateThrottle',
        'rest_framework.throttling.UserRateThrottle'
    ),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.TokenAuthentication',
        'rest_framework.authentication.SessionAuthentication',
    ),
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 250,
}

# Shell Plus
SHELL_PLUS_DONT_LOAD = ['tosec']

SILENCED_SYSTEM_CHECKS = [
    'urls.W002',
]

# Logging
SEND_BROKEN_LINK_EMAILS = False
LOGGING_HANDLERS = ['file', 'mail_admins', 'console']
DEFAULT_LOGGING_CONFIG = {
    'handlers': LOGGING_HANDLERS,
    'level': 'DEBUG',
    'propagate': True,
}
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s '
                      '%(process)d %(thread)d %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        }
    },
    'handlers': {
        'null': {
            'level': 'DEBUG',
            'class': 'logging.NullHandler',
        },
        'mail_admins': {
            'level': 'ERROR',
            'include_html': True,
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'formatter': 'verbose',
            'filename': 'lutrisweb.log'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': LOGGING_HANDLERS,
            'propagate': True,
            'level': 'INFO',
        },
        'scripts': {
            'handlers': LOGGING_HANDLERS,
            'propagate': True,
            'level': 'DEBUG',
        },
        'factory': {
            'handlers': ['null'],
            'propagate': False,
            'level': 'INFO',
        },
        'django.request': {
            'handlers': LOGGING_HANDLERS,
            'level': 'WARNING',
            'propagate': True,
        },
        'lutrisweb': DEFAULT_LOGGING_CONFIG,
        'accounts': DEFAULT_LOGGING_CONFIG,
        'common': DEFAULT_LOGGING_CONFIG,
        'games': DEFAULT_LOGGING_CONFIG,
        'thegamesdb': DEFAULT_LOGGING_CONFIG
    }
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
configuration.py

Created by Thomas Mangin on 2011-11-29.
Copyright (c) 2011 Exa Networks. All rights reserved.
"""

# XXX: raised exception not caught
# XXX: reloading mid-program not possible
# XXX: validation for path, file, etc not correctly test (ie surely buggy)

import os
import sys
import syslog
import pwd

from .util.logger import logger
from .util.version import version

class ConfigurationError (Exception):
	pass

_priorities = {
	'LOG_EMERG'    : syslog.LOG_EMERG,
	'LOG_ALERT'    : syslog.LOG_ALERT,
	'LOG_CRIT'     : syslog.LOG_CRIT,
	'LOG_ERR'      : syslog.LOG_ERR,
	'LOG_WARNING'  : syslog.LOG_WARNING,
	'LOG_NOTICE'   : syslog.LOG_NOTICE,
	'LOG_INFO'     : syslog.LOG_INFO,
	'LOG_DEBUG'    : syslog.LOG_DEBUG,
}

#class log:
#	LOG_DEBUG    = syslog.LOG_DEBUG
#	LOG_INFO     = syslog.LOG_INFO
#	LOG_NOTICE   = syslog.LOG_NOTICE
#	LOG_WARNING  = syslog.LOG_WARNING
#	LOG_ERR      = syslog.LOG_ERR
#	LOG_CRIT     = syslog.LOG_CRIT
#	LOG_ALERT    = syslog.LOG_ALERT
#	LOG_EMERG    = syslog.LOG_EMERG

class NoneDict (dict):
	def __getitem__ (self,name):
		return None
nonedict = NoneDict()

class value (object):
	location = os.path.normpath(sys.argv[0]) if sys.argv[0].startswith('/') else os.path.normpath(os.path.join(cwd,sys.argv[0]))

	@staticmethod
	def integer (_):
		return int(_)

	@staticmethod
	def unquote (_):
		 return _.strip().strip('\'"')

	@staticmethod
	def nop (_):
		return _

	@staticmethod
	def boolean (_):
		return _.lower() in ('1','yes','on','enable','true')

	@staticmethod
	def user (_):
		# XXX: incomplete
		try:
			answer = pwd.getpwnam(_)
			# uid = answer[2]
		except KeyError:
			raise TypeError('user %s is not found on this system' % _)
		return _

	@staticmethod
	def folder(path):
		path = value.unquote(path)
		paths = [
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),path)),
			os.path.normpath(os.path.join('/','etc','exaproxy','exaproxy.conf',path)),
			os.path.normpath(path)
		]
		options = [path for path in paths if os.path.exists(path)]
		if not options: raise TypeError('%s does not exists' % path)
		first = options[0]
		if not first: raise TypeError('%s does not exists' % first)
		return first

	@staticmethod
	def conf(path):
		first = value.folder(path)
		if not os.path.isfile(first): raise TypeError('%s is not a file' % path)
		return first

	@staticmethod
	def resolver(path):
		paths = [
			os.path.normpath(path),
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),path)),
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),'etc','exaproxy','resolv.conf')),
			os.path.normpath(os.path.join('/','etc','exaproxy','resolv.conf',path)),
		]
		for resolver in paths:
			if os.path.exists(resolver):
				with open(resolver) as r:
					if 'nameserver' in (line.strip().split(None,1)[0].lower() for line in r.readlines()):
						return resolver
		raise TypeError('resolv.conf can not be found')

	@staticmethod
	def exe (path):
		first = value.conf(path)
		if not os.access(first, os.X_OK): raise TypeError('%s is not an executable' % first)
		return first

	@staticmethod
	def syslog (log):
		if log not in _priorities:
			raise TypeError('invalid log level %s' % log)
		return _priorities[log]

defaults = {
	'tcp4' : {
		'host'    : (value.unquote,'127.0.0.1'   , 'the host the proxy listen on'),
		'port'    : (value.integer,'31280'       , 'the port the proxy listen on'),
		'timeout' : (value.integer,'5'           , 'time before we ...'),
		'backlog' : (value.integer,'200'         , 'when busy how many connection should the OS keep for us'),
		'listen'  : (value.boolean,'true'        , 'should we listen for connections over IPv4'),
		'out'     : (value.boolean,'true'        , 'allow connections to remote web servers over IPv4'),
	},
	'tcp6' : {
		'host'    : (value.unquote,'::1'         , 'the host the proxy listen on'),
		'port'    : (value.integer,'31280'       , 'the port the proxy listen on'),
		'timeout' : (value.integer,'5'           , 'time before we ...'),
		'backlog' : (value.integer,'200'         , 'when busy how many connection should the OS keep for us'),
		'listen'  : (value.boolean,'false'       , 'should we listen for connections over IPv6'),
		'out'     : (value.boolean,'true'        , 'allow connections to remote web servers over IPv6'),
	},
	'redirector' : {
		'program' : (value.exe,'etc/exaproxy/redirector/allow'  , 'the program used to know where to send request'),
		'minimum' : (value.integer,'5'                          , 'minimum number of worker threads (forked program)'),
		'maximum' : (value.integer,'25'                         , 'maximum number of worker threads (forked program)'),
#		'timeout' : (value.integer,'1'                          , 'how long to wait for work before peforming background work'),
	},

	'http' : {
		'x-forwarded-for' : (value.boolean,'true'    , 'insert x-forarded-for headers to webservers'),
		'allow-connect'   : (value.boolean,'true'    , 'allow client to use CONNECT and https connections'),
	},
	'web' : {
		'enabled' : (value.boolean,'true'              , 'enable the built-in webserver'),
		'port'    : (value.integer,'8080'              , 'port on which the web server listen'),
		'html'    : (value.folder,'etc/exaproxy/html'  , 'where are the proxy served pages are taken from'),
	},
	'daemon' : {
		'pidfile'     : (value.unquote,''                    , 'where to save the pid if we manage it'),
		'user'        : (value.user,'nobody'                 , 'user to run as'),
		'daemonise'   : (value.boolean,'false'               , 'should we run in the background'),
		'reactor'     : (value.unquote,'epoll'               , 'what event mechanism to use (select/epoll)'),
		'speed'       : (value.integer,'2'                   , 'when waiting for connection how long are we sleeping for'),
	},
	'dns' : {
		'resolver' : (value.resolver,'/etc/resolv.conf'   , 'resolver file'),
		'timeout'  : (value.integer,'2'                   , 'how long to wait for DNS replies'),
	},
	'logger' : {
		'level'         : (value.syslog,'LOG_ERR'  , 'log message with at least the priority SYSLOG.<level>'),
		'destination'   : (value.unquote,'stdout'  , 'where syslog should log'),
		'logger'        : (value.boolean,'true'    , 'log message from the logger subsystem'),
		'signal'        : (value.boolean,'true'    , 'log message from the signal subsystem'),
		'configuration' : (value.boolean,'true'    , 'log message from the configuration subsystem'),
		'supervisor'    : (value.boolean,'true'    , 'log message from the supervisor subsystem'),
		'daemon'        : (value.boolean,'true'    , 'log message from the daemon subsystem'),
		'server'        : (value.boolean,'true'    , 'log message from the server subsystem'),
		'client'        : (value.boolean,'true'    , 'log message from the client subsystem'),
		'manager'       : (value.boolean,'true'    , 'log message from the manager subsystem'),
		'worker'        : (value.boolean,'true'    , 'log message from the worker subsystem'),
		'download'      : (value.boolean,'true'    , 'log message from the download subsystem'),
		'http'          : (value.boolean,'true'    , 'log message from the http subsystem'),
		'client'        : (value.boolean,'true'    , 'log message from the client subsystem'),
	},
	'profile' : {
		'enabled'     : (value.boolean,'false'  , 'enable profiling'),
		'destination' : (value.nop,'stdout'     , 'save profiling to file (instead to the screen on exits)'),
	},
	'proxy' : {
		'name'    : (value.nop,'ExaProxy'   , 'name'),
		'version' : (value.nop,version      , 'version'),
	}
}

import ConfigParser

class Store (dict):
	def __getitem__ (self,key):
		return dict.__getitem__(self,key.replace('_','-'))
	def __setitem__ (self,key,value):
		return dict.__setitem__(self,key.replace('_','-'),value)
	def __getattr__ (self,key):
		return dict.__getitem__(self,key.replace('_','-'))
	def __setattr__ (self,key,value):
		return dict.__setitem__(self,key.replace('_','-'),value)

def _configuration ():
	_conf_paths = (
		os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),'etc','exaproxy','exaproxy.conf')),
		os.path.normpath(os.path.join('/','etc','exaproxy','exaproxy.conf')),
	)

	ini_file = [path for path in _conf_paths if os.path.exists(path)][0]
	if not ini_file:
		raise ConfigurationError('could not find exaproxy.conf file')

	ini = ConfigParser.ConfigParser()
	ini.read(ini_file)

	configuration = Store()

	for section in defaults:
		default = defaults[section]

		for option in default:
			convert = default[option][0]
			try:
				proxy_section = 'exaproxy.%s' % section
				env_name = '%s.%s' % (proxy_section,option)
				conf = value.unquote(os.environ.get(env_name,'')) \
				    or value.unquote(os.environ.get(env_name.replace('.','_'),'')) \
				    or value.unquote(ini.get(proxy_section,option,nonedict)) \
				    or default[option][1]
			except (ConfigParser.NoSectionError,ConfigParser.NoOptionError),e:
				conf = default[option][1]
			try:
				configuration.setdefault(section,Store())[option] = convert(conf)
			except TypeError,e:
				raise ConfigurationError('invalid value for %s.%s : %s' % (section,option,conf))

	return configuration

__configuration = None

def load ():
	global __configuration
	if __configuration:
		return __configuration
	__configuration = _configuration()
	return __configuration

def default ():
	for section,content in defaults.items():
		if section == 'proxy':
			continue
		for option,value in content.items():
			if option == 'proxy':
				continue
			yield 'exaproxy.%s.%s %s: %s. default (%s)' % (section,option,' '*(20-len(section)-len(option)),value[2],value[1])

def ini ():
	for section,values in __configuration.items():
		if section == 'proxy':
			continue
		print '[exaproxy.%s]' % section
		for k,v in values.items():
			print '%s = %s' % (k,v)
		print

def env ():
	for section,values in __configuration.items():
		if section == 'proxy':
			continue
		for k,v in values.items():
			print 'exaproxy.%s.%s="%s"' % (section,k,v)
	print
<commit_msg>Add config options for dns cache expiry<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
configuration.py

Created by Thomas Mangin on 2011-11-29.
Copyright (c) 2011 Exa Networks. All rights reserved.
"""

# XXX: raised exception not caught
# XXX: reloading mid-program not possible
# XXX: validation for path, file, etc not correctly test (ie surely buggy)

import os
import sys
import syslog
import pwd

from .util.logger import logger
from .util.version import version

class ConfigurationError (Exception):
	pass

_priorities = {
	'LOG_EMERG'    : syslog.LOG_EMERG,
	'LOG_ALERT'    : syslog.LOG_ALERT,
	'LOG_CRIT'     : syslog.LOG_CRIT,
	'LOG_ERR'      : syslog.LOG_ERR,
	'LOG_WARNING'  : syslog.LOG_WARNING,
	'LOG_NOTICE'   : syslog.LOG_NOTICE,
	'LOG_INFO'     : syslog.LOG_INFO,
	'LOG_DEBUG'    : syslog.LOG_DEBUG,
}

#class log:
#	LOG_DEBUG    = syslog.LOG_DEBUG
#	LOG_INFO     = syslog.LOG_INFO
#	LOG_NOTICE   = syslog.LOG_NOTICE
#	LOG_WARNING  = syslog.LOG_WARNING
#	LOG_ERR      = syslog.LOG_ERR
#	LOG_CRIT     = syslog.LOG_CRIT
#	LOG_ALERT    = syslog.LOG_ALERT
#	LOG_EMERG    = syslog.LOG_EMERG

class NoneDict (dict):
	def __getitem__ (self,name):
		return None
nonedict = NoneDict()

class value (object):
	location = os.path.normpath(sys.argv[0]) if sys.argv[0].startswith('/') else os.path.normpath(os.path.join(cwd,sys.argv[0]))

	@staticmethod
	def integer (_):
		return int(_)

	@staticmethod
	def unquote (_):
		 return _.strip().strip('\'"')

	@staticmethod
	def nop (_):
		return _

	@staticmethod
	def boolean (_):
		return _.lower() in ('1','yes','on','enable','true')

	@staticmethod
	def user (_):
		# XXX: incomplete
		try:
			answer = pwd.getpwnam(_)
			# uid = answer[2]
		except KeyError:
			raise TypeError('user %s is not found on this system' % _)
		return _

	@staticmethod
	def folder(path):
		path = value.unquote(path)
		paths = [
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),path)),
			os.path.normpath(os.path.join('/','etc','exaproxy','exaproxy.conf',path)),
			os.path.normpath(path)
		]
		options = [path for path in paths if os.path.exists(path)]
		if not options: raise TypeError('%s does not exists' % path)
		first = options[0]
		if not first: raise TypeError('%s does not exists' % first)
		return first

	@staticmethod
	def conf(path):
		first = value.folder(path)
		if not os.path.isfile(first): raise TypeError('%s is not a file' % path)
		return first

	@staticmethod
	def resolver(path):
		paths = [
			os.path.normpath(path),
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),path)),
			os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),'etc','exaproxy','resolv.conf')),
			os.path.normpath(os.path.join('/','etc','exaproxy','resolv.conf',path)),
		]
		for resolver in paths:
			if os.path.exists(resolver):
				with open(resolver) as r:
					if 'nameserver' in (line.strip().split(None,1)[0].lower() for line in r.readlines()):
						return resolver
		raise TypeError('resolv.conf can not be found')

	@staticmethod
	def exe (path):
		first = value.conf(path)
		if not os.access(first, os.X_OK): raise TypeError('%s is not an executable' % first)
		return first

	@staticmethod
	def syslog (log):
		if log not in _priorities:
			raise TypeError('invalid log level %s' % log)
		return _priorities[log]

defaults = {
	'tcp4' : {
		'host'    : (value.unquote,'127.0.0.1'   , 'the host the proxy listen on'),
		'port'    : (value.integer,'31280'       , 'the port the proxy listen on'),
		'timeout' : (value.integer,'5'           , 'time before we ...'),
		'backlog' : (value.integer,'200'         , 'when busy how many connection should the OS keep for us'),
		'listen'  : (value.boolean,'true'        , 'should we listen for connections over IPv4'),
		'out'     : (value.boolean,'true'        , 'allow connections to remote web servers over IPv4'),
	},
	'tcp6' : {
		'host'    : (value.unquote,'::1'         , 'the host the proxy listen on'),
		'port'    : (value.integer,'31280'       , 'the port the proxy listen on'),
		'timeout' : (value.integer,'5'           , 'time before we ...'),
		'backlog' : (value.integer,'200'         , 'when busy how many connection should the OS keep for us'),
		'listen'  : (value.boolean,'false'       , 'should we listen for connections over IPv6'),
		'out'     : (value.boolean,'true'        , 'allow connections to remote web servers over IPv6'),
	},
	'redirector' : {
		'program' : (value.exe,'etc/exaproxy/redirector/allow'  , 'the program used to know where to send request'),
		'minimum' : (value.integer,'5'                          , 'minimum number of worker threads (forked program)'),
		'maximum' : (value.integer,'25'                         , 'maximum number of worker threads (forked program)'),
#		'timeout' : (value.integer,'1'                          , 'how long to wait for work before peforming background work'),
	},

	'http' : {
		'x-forwarded-for' : (value.boolean,'true'    , 'insert x-forarded-for headers to webservers'),
		'allow-connect'   : (value.boolean,'true'    , 'allow client to use CONNECT and https connections'),
	},
	'web' : {
		'enabled' : (value.boolean,'true'              , 'enable the built-in webserver'),
		'port'    : (value.integer,'8080'              , 'port on which the web server listen'),
		'html'    : (value.folder,'etc/exaproxy/html'  , 'where are the proxy served pages are taken from'),
	},
	'daemon' : {
		'pidfile'     : (value.unquote,''                    , 'where to save the pid if we manage it'),
		'user'        : (value.user,'nobody'                 , 'user to run as'),
		'daemonise'   : (value.boolean,'false'               , 'should we run in the background'),
		'reactor'     : (value.unquote,'epoll'               , 'what event mechanism to use (select/epoll)'),
		'speed'       : (value.integer,'2'                   , 'when waiting for connection how long are we sleeping for'),
	},
	'dns' : {
		'resolver' : (value.resolver,'/etc/resolv.conf'   , 'resolver file'),
		'timeout'  : (value.integer,'2'                   , 'how long to wait for DNS replies'),
		'cache_expiry' : (value.integer,'120'             , 'amount of time we will cache dns results for'),
		'expire_items' : (value.integer,'200'             , 'max number of cached items we will expire during cleanup'),
	},
	'logger' : {
		'level'         : (value.syslog,'LOG_ERR'  , 'log message with at least the priority SYSLOG.<level>'),
		'destination'   : (value.unquote,'stdout'  , 'where syslog should log'),
		'logger'        : (value.boolean,'true'    , 'log message from the logger subsystem'),
		'signal'        : (value.boolean,'true'    , 'log message from the signal subsystem'),
		'configuration' : (value.boolean,'true'    , 'log message from the configuration subsystem'),
		'supervisor'    : (value.boolean,'true'    , 'log message from the supervisor subsystem'),
		'daemon'        : (value.boolean,'true'    , 'log message from the daemon subsystem'),
		'server'        : (value.boolean,'true'    , 'log message from the server subsystem'),
		'client'        : (value.boolean,'true'    , 'log message from the client subsystem'),
		'manager'       : (value.boolean,'true'    , 'log message from the manager subsystem'),
		'worker'        : (value.boolean,'true'    , 'log message from the worker subsystem'),
		'download'      : (value.boolean,'true'    , 'log message from the download subsystem'),
		'http'          : (value.boolean,'true'    , 'log message from the http subsystem'),
		'client'        : (value.boolean,'true'    , 'log message from the client subsystem'),
	},
	'profile' : {
		'enabled'     : (value.boolean,'false'  , 'enable profiling'),
		'destination' : (value.nop,'stdout'     , 'save profiling to file (instead to the screen on exits)'),
	},
	'proxy' : {
		'name'    : (value.nop,'ExaProxy'   , 'name'),
		'version' : (value.nop,version      , 'version'),
	}
}

import ConfigParser

class Store (dict):
	def __getitem__ (self,key):
		return dict.__getitem__(self,key.replace('_','-'))
	def __setitem__ (self,key,value):
		return dict.__setitem__(self,key.replace('_','-'),value)
	def __getattr__ (self,key):
		return dict.__getitem__(self,key.replace('_','-'))
	def __setattr__ (self,key,value):
		return dict.__setitem__(self,key.replace('_','-'),value)

def _configuration ():
	_conf_paths = (
		os.path.normpath(os.path.join(os.path.join(os.sep,*os.path.join(value.location.split(os.sep)[:-3])),'etc','exaproxy','exaproxy.conf')),
		os.path.normpath(os.path.join('/','etc','exaproxy','exaproxy.conf')),
	)

	ini_file = [path for path in _conf_paths if os.path.exists(path)][0]
	if not ini_file:
		raise ConfigurationError('could not find exaproxy.conf file')

	ini = ConfigParser.ConfigParser()
	ini.read(ini_file)

	configuration = Store()

	for section in defaults:
		default = defaults[section]

		for option in default:
			convert = default[option][0]
			try:
				proxy_section = 'exaproxy.%s' % section
				env_name = '%s.%s' % (proxy_section,option)
				conf = value.unquote(os.environ.get(env_name,'')) \
				    or value.unquote(os.environ.get(env_name.replace('.','_'),'')) \
				    or value.unquote(ini.get(proxy_section,option,nonedict)) \
				    or default[option][1]
			except (ConfigParser.NoSectionError,ConfigParser.NoOptionError),e:
				conf = default[option][1]
			try:
				configuration.setdefault(section,Store())[option] = convert(conf)
			except TypeError,e:
				raise ConfigurationError('invalid value for %s.%s : %s' % (section,option,conf))

	return configuration

__configuration = None

def load ():
	global __configuration
	if __configuration:
		return __configuration
	__configuration = _configuration()
	return __configuration

def default ():
	for section,content in defaults.items():
		if section == 'proxy':
			continue
		for option,value in content.items():
			if option == 'proxy':
				continue
			yield 'exaproxy.%s.%s %s: %s. default (%s)' % (section,option,' '*(20-len(section)-len(option)),value[2],value[1])

def ini ():
	for section,values in __configuration.items():
		if section == 'proxy':
			continue
		print '[exaproxy.%s]' % section
		for k,v in values.items():
			print '%s = %s' % (k,v)
		print

def env ():
	for section,values in __configuration.items():
		if section == 'proxy':
			continue
		for k,v in values.items():
			print 'exaproxy.%s.%s="%s"' % (section,k,v)
	print
<|endoftext|>


<commit_before>import numpy as np

######
# Computes the viterbi path for a given sequence of lenght.
# N - Lenght of sequence
# H - Number of hidden states
# Receives:
# Node potentials (N,H) vector
# Edge potentials (N-1,H,H)
######
def viterbi(node_potentials,edge_potentials):
  # Implement your function here.
<commit_msg>corrected small typo in our template for viterbi.<commit_after>import numpy as np

######
# Computes the viterbi path for a given sequence of lenght.
# N - Lenght of sequence
# H - Number of hidden states
# Receives:
# Node potentials (N,H) vector
# Edge potentials (N-1,H,H)
######
def viterbi(node_potentials,edge_potentials):
  # Implement your function here.
  pass
<|endoftext|>


<commit_before>import os
import csv
import json
import logging
import threading
from datetime import datetime, timedelta
import sys

import requests
import pandas as pd
from kafka import KafkaProducer
from logstash import TCPLogstashHandler

CM_APP_HOST = 'http://192.168.13.101'
KAFKA_TOPIC = 'SensorData'
BOOTSTRAP_SERVERS = ['il061:9092', 'il062:9092', 'il063:9092']
IGNORED_FIELDS = ['Record', 'Wind', 'Temp_Aussen', 'Feuchte_Aussen']
UPDATE_INTERVAL = 1  # in minutes

LOGSTASH_HOST = os.getenv('LOGSTASH_HOST', 'localhost')
LOGSTASH_PORT = int(os.getenv('LOGSTASH_PORT', '5000'))

SENSORTHINGS_HOST = os.getenv('SENSORTHINGS_HOST', 'il060')
SENSORTHINGS_PORT = os.getenv('SENSORTHINGS_PORT', '8082')

# setup logging
logger = logging.getLogger('cm-adapter')
logger.setLevel(logging.INFO)
console_logger = logging.StreamHandler(stream=sys.stdout)
console_logger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logstash_handler = TCPLogstashHandler(host=LOGSTASH_HOST, port=LOGSTASH_PORT, version=1)
[logger.addHandler(l) for l in [console_logger, logstash_handler]]
logger.info('Sending logstash to %s:%d', logstash_handler.host, logstash_handler.port)

producer = KafkaProducer(bootstrap_servers=BOOTSTRAP_SERVERS,
                         api_version=(0, 9),
                         value_serializer=lambda m: json.dumps(m).encode('ascii'),
                         acks='all')

def update(last_sent_time=None, id_map=None):
    """
    Fetches recent sensor data from the CM setup and forwards new entries to the i-Maintenance messaging bus.
    After fetching and updating data a new cycle is scheduled.
    :param last_sent_time: Last time of previous update. Used to determine new entries.
        If `None`, all entries will be forwarded.
    :param id_map: Given mapping of SensorThings IDs. This map is updated during the iteration.
    """
    try:
        # fetch sensor data
        sensor_data = fetch_sensor_data(cm_host=CM_APP_HOST)
        logger.info('Fetched {} sensor entries'.format(len(sensor_data)))

        # filter data
        sensor_data = sensor_data.ix[sensor_data.index > last_sent_time] if last_sent_time else sensor_data

        # fetch id mapping
        id_map = fetch_id_mapping(host=SENSORTHINGS_HOST, port=SENSORTHINGS_PORT, fallback=id_map)

        # delegate to messaging bus
        publish_sensor_data(data=sensor_data, id_map=id_map, topic=KAFKA_TOPIC, ignored=IGNORED_FIELDS)
        last_sent_time = sensor_data.index[-1]
        logger.info('Published {} new sensor entries till {}'.format(len(sensor_data), last_sent_time))

    except Exception as e:
        logger.exception(e)

    # schedule next update
    interval = timedelta(minutes=UPDATE_INTERVAL).total_seconds()
    kwargs = {'last_sent_time': last_sent_time, 'id_map': id_map}
    threading.Timer(interval=interval, function=update, kwargs=kwargs).start()
    logger.info('Scheduled next update at {}'.format(datetime.now() + timedelta(minutes=UPDATE_INTERVAL)))

def fetch_sensor_data(cm_host):
    """
    Fetches sensor data from the CM host.
    :param cm_host: URL of CH host.
    :return: Dataframe containing fetched entries, whereas the column matches the columns in the CSV file.
    """
    url = cm_host + '/FileBrowser/Download?Path=/DataLogs/SalzburgResearch_Logging.csv'
    headers = {'Referer': cm_host + '/Portal/Portal.mwsl?PriNav=FileBrowser&Path=/DataLogs/"'}
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    csv_data = response.text.splitlines()

    # read sensor data from csv file
    csv_reader = csv.reader(csv_data)
    csv_header = next(csv_reader)  # read header information

    sensor_data = pd.DataFrame(columns=csv_header)
    for row in csv_reader:
        if row:  # due to blank line at the bottom
            sensor_data.loc[row[0]] = list(row)  # index is first field (i.e. 'Record')

    # convert timestamp
    sensor_data['Zeitstempel'] = pd.to_datetime(sensor_data['Zeitstempel'])
    sensor_data = sensor_data.set_index(['Zeitstempel']).sort_index()

    return sensor_data

def publish_sensor_data(data, id_map, topic, ignored=None):
    """
    Published sensor data to the i-Maintenance messaging bus (Kafka) using the SensorThings format.
    :param data: Dataframe containing sensor data.
    :param id_map: `dict` of mapping columns in the Dataframe to IDs in the SensorThings domain.
    :param topic: Kafka topic in which the messages are published.
    :param ignored: List of ignored column names. If `None` rows of all columns are published.
    """
    if ignored is None:
        ignored = []

    for observation_time in data.index:
        for sensor in [s for s in data.columns if s not in ignored]:
            message = {'phenomenonTime': observation_time.isoformat(),
                       'resultTime': datetime.now().isoformat(),
                       'result': float(data.loc[observation_time, sensor]),
                       'Datastream': {'@iot.id': id_map[sensor]}}
            producer.send(topic, message)

    # block until all messages are sent
    producer.flush()

def fetch_id_mapping(host, port, fallback):
    """
    Fetches IDs from SensorThings server and creates a dictionary with the proper ID mapping.
    :param host: Host of SensorThings server.
    :param port: Port of SensorThings server.
    :param fallback: Fallback mapping in case of an error. If `None` the actual error while be raised.
    :return: `dict`, which is mapping CM specific IDs to global SensorThings IDs.
    """
    mapping = dict()
    try:
        url = 'http://{}:{}/v1.0/Datastreams'.format(host, port)
        while True:
            url = url.replace('localhost', host).replace('8080', port)  # replace wrong base url and port
            datastreams = requests.get(url=url).json()

            mapping.update({d['name']: d['@iot.id'] for d in datastreams['value']})

            if '@iot.nextLink' not in datastreams:
                break
            url = datastreams['@iot.nextLink']

        logger.info('Fetched id mapping: %s', mapping, extra={'': mapping})
    except Exception as e:
        if not fallback:
            raise e
        logger.warning('Could not fetch id mapping...')
        return fallback

    return mapping

if __name__ == '__main__':
    update()
<commit_msg>minor refactoring<commit_after>import os
import csv
import json
import logging
import threading
from datetime import datetime, timedelta
import sys

import requests
import pandas as pd
from kafka import KafkaProducer
from logstash import TCPLogstashHandler

CM_APP_HOST = 'http://192.168.13.101'
KAFKA_TOPIC = 'SensorData'
BOOTSTRAP_SERVERS = ['il061:9092', 'il062:9092', 'il063:9092']
IGNORED_FIELDS = ['Record', 'Wind', 'Temp_Aussen', 'Feuchte_Aussen']
UPDATE_INTERVAL = 1  # in minutes

LOGSTASH_HOST = os.getenv('LOGSTASH_HOST', 'localhost')
LOGSTASH_PORT = int(os.getenv('LOGSTASH_PORT', '5000'))

SENSORTHINGS_HOST = os.getenv('SENSORTHINGS_HOST', 'il060')
SENSORTHINGS_PORT = os.getenv('SENSORTHINGS_PORT', '8082')

# setup logging
logger = logging.getLogger('cm-adapter')
logger.setLevel(logging.INFO)
console_logger = logging.StreamHandler(stream=sys.stdout)
console_logger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logstash_handler = TCPLogstashHandler(host=LOGSTASH_HOST, port=LOGSTASH_PORT, version=1)
[logger.addHandler(l) for l in [console_logger, logstash_handler]]
logger.info('Sending logstash to %s:%d', logstash_handler.host, logstash_handler.port)

producer = KafkaProducer(bootstrap_servers=BOOTSTRAP_SERVERS,
                         api_version=(0, 9),
                         value_serializer=lambda m: json.dumps(m).encode('ascii'),
                         acks='all')

def update(last_sent_time=None, id_map=None):
    """
    Fetches recent sensor data from the CM setup and forwards new entries to the i-Maintenance messaging bus.
    After fetching and updating data a new cycle is scheduled.
    :param last_sent_time: Last time of previous update. Used to determine new entries.
        If `None`, all entries will be forwarded.
    :param id_map: Given mapping of SensorThings IDs. This map is updated during the iteration.
    """
    try:
        # fetch sensor data
        sensor_data = fetch_sensor_data(cm_host=CM_APP_HOST)
        logger.info('Fetched {} sensor entries'.format(len(sensor_data)))

        # filter data
        sensor_data = sensor_data.ix[sensor_data.index > last_sent_time] if last_sent_time else sensor_data

        # fetch id mapping
        id_map = fetch_id_mapping(host=SENSORTHINGS_HOST, port=SENSORTHINGS_PORT, fallback=id_map)

        # delegate to messaging bus
        publish_sensor_data(data=sensor_data, id_map=id_map, topic=KAFKA_TOPIC, ignored=IGNORED_FIELDS)
        last_sent_time = sensor_data.index[-1]
        logger.info('Published {} new sensor entries till {}'.format(len(sensor_data), last_sent_time))

    except Exception as e:
        logger.exception(e)

    # schedule next update
    interval = timedelta(minutes=UPDATE_INTERVAL).total_seconds()
    kwargs = {'last_sent_time': last_sent_time, 'id_map': id_map}
    threading.Timer(interval=interval, function=update, kwargs=kwargs).start()
    logger.info('Scheduled next update at {}'.format(datetime.now() + timedelta(minutes=UPDATE_INTERVAL)))

def fetch_sensor_data(cm_host):
    """
    Fetches sensor data from the CM host.
    :param cm_host: URL of CH host.
    :return: Dataframe containing fetched entries, whereas the column matches the columns in the CSV file.
    """
    url = cm_host + '/FileBrowser/Download?Path=/DataLogs/SalzburgResearch_Logging.csv'
    headers = {'Referer': cm_host + '/Portal/Portal.mwsl?PriNav=FileBrowser&Path=/DataLogs/"'}
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    csv_data = response.text.splitlines()

    # read sensor data from csv file
    csv_reader = csv.reader(csv_data)
    csv_header = next(csv_reader)  # read header information

    sensor_data = pd.DataFrame(columns=csv_header)
    for row in csv_reader:
        if row:  # due to blank line at the bottom
            sensor_data.loc[row[0]] = list(row)  # index is first field (i.e. 'Record')

    # convert timestamp
    sensor_data['Zeitstempel'] = pd.to_datetime(sensor_data['Zeitstempel'])
    sensor_data = sensor_data.set_index(['Zeitstempel']).sort_index()

    return sensor_data

def publish_sensor_data(data, id_map, topic, ignored=None):
    """
    Published sensor data to the i-Maintenance messaging bus (Kafka) using the SensorThings format.
    :param data: Dataframe containing sensor data.
    :param id_map: `dict` of mapping columns in the Dataframe to IDs in the SensorThings domain.
    :param topic: Kafka topic in which the messages are published.
    :param ignored: List of ignored column names. If `None` rows of all columns are published.
    """
    if ignored is None:
        ignored = []

    for observation_time in data.index:
        for sensor in [s for s in data.columns if s not in ignored]:
            message = {'phenomenonTime': observation_time.isoformat(),
                       'resultTime': datetime.now().isoformat(),
                       'result': float(data.loc[observation_time, sensor]),
                       'Datastream': {'@iot.id': id_map[sensor]}}
            producer.send(topic, message)

    # block until all messages are sent
    producer.flush()

def fetch_id_mapping(host, port, fallback):
    """
    Fetches IDs from SensorThings server and creates a dictionary with the proper ID mapping.
    :param host: Host of SensorThings server.
    :param port: Port of SensorThings server.
    :param fallback: Fallback mapping in case of an error. If `None` the actual error while be raised.
    :return: `dict`, which is mapping CM specific IDs to global SensorThings IDs.
    """
    mapping = dict()
    try:
        url = 'http://{}:{}/v1.0/Datastreams'.format(host, port)
        while True:
            url = url.replace('localhost', host).replace('8080', port)  # replace wrong base url and port
            datastreams = requests.get(url=url).json()

            mapping.update({d['name']: d['@iot.id'] for d in datastreams['value']})

            if '@iot.nextLink' not in datastreams:
                break
            url = datastreams['@iot.nextLink']

        logger.info('Fetched id mapping: %s', mapping, extra={'sensorthings_ids': mapping})
    except Exception as e:
        if not fallback:
            raise e
        logger.warning('Could not fetch id mapping...')
        return fallback

    return mapping

if __name__ == '__main__':
    update()
<|endoftext|>