<commit_before>#Create EC2 instance, EBS, and RDS alarms for all resources in a region

import boto3
import collections

#SNS Topic Definition for EC2, EBS, and RDS
ec2_sns = 'SNS-ARN'
ebs_sns = 'SNS-ARN'
rds_sns = 'SNS-ARN'

ec = boto3.client('ec2')
rd = boto3.client('rds')
cw = boto3.client('cloudwatch')

def lambda_handler(event, context):
    reservations = ec.describe_instances().get('Reservations', [])
    instances = sum(
        [
            [i for i in r['Instances']]
            for r in reservations
        ], [])

    for instance in instances:
        for tag in instance['Tags']:
            if tag['Key'] == 'Name':
                name_tag = tag['Value']
                print "Found instance %s with name %s" % (instance['InstanceId'], name_tag)

        #Create Metric "CPU Utilization Greater than 95% for 15+ Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High CPU Utilization Warning" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Utilization Greater than 95% for 15+ Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUUtilization',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=3,
            Threshold=95.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

        #Create Metric "CPU Utilization Greater than 95% for 60+ Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High CPU Utilization Critical" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Utilization Greater than 95% for 60+ Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUUtilization',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=12,
            Threshold=95.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

        #Create Metric "CPU Credit Balance <= 25 for 30 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s Credit Balance Warning" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Credit Balance <= 25 for 30 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUCreditBalance',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=6,
            Threshold=25.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

        #Create Metric "CPU Credit Balance <= 5 for 10 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s Credit Balance Critical" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Credit Balance <= 5 for 10 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUCreditBalance',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=2,
            Threshold=5.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

    for instance in instances:
        for dev in instance['BlockDeviceMappings']:
            if dev.get('Ebs', None) is None:
                continue
            vol_id = dev['Ebs']['VolumeId']
            print "Found EBS volume %s on instance %s" % (
                vol_id, instance['InstanceId'])

    database_instances = rd.describe_db_instances().get('DBInstances', [])
    for database in database_instances:
            print "Found RDS database %s" % (
                 database['DBInstanceIdentifier'])

    metricalarms = cw.describe_alarms().get('MetricAlarms', [])
    for alarm in metricalarms:
            print "Found CloudWatch alarm %s" % (
                 alarm['AlarmName'])
<commit_msg>Added Volume Metrics<commit_after>#Create EC2 instance, EBS, and RDS alarms for all resources in a region

import boto3
import collections

#SNS Topic Definition for EC2, EBS, and RDS
ec2_sns = 'SNS-ARN'
ebs_sns = 'SNS-ARN'
rds_sns = 'SNS-ARN'

ec = boto3.client('ec2')
rd = boto3.client('rds')
cw = boto3.client('cloudwatch')

def lambda_handler(event, context):
    reservations = ec.describe_instances().get('Reservations', [])
    instances = sum(
        [
            [i for i in r['Instances']]
            for r in reservations
        ], [])

    for instance in instances:
        for tag in instance['Tags']:
            if tag['Key'] == 'Name':
                name_tag = tag['Value']
                print "Found instance %s with name %s" % (instance['InstanceId'], name_tag)

        #Create Metric "CPU Utilization Greater than 95% for 15+ Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High CPU Utilization Warning" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Utilization Greater than 95% for 15+ Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUUtilization',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=3,
            Threshold=95.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

        #Create Metric "CPU Utilization Greater than 95% for 60+ Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High CPU Utilization Critical" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Utilization Greater than 95% for 60+ Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUUtilization',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=12,
            Threshold=95.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

        #Create Metric "CPU Credit Balance <= 25 for 30 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s Credit Balance Warning" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Credit Balance <= 25 for 30 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUCreditBalance',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=6,
            Threshold=25.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

        #Create Metric "CPU Credit Balance <= 5 for 10 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s Credit Balance Critical" % (name_tag, instance['InstanceId']),
            AlarmDescription='CPU Credit Balance <= 5 for 10 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='CPUCreditBalance',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=300,
            EvaluationPeriods=2,
            Threshold=5.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

        #Create Metric "Status Check Failed (System) for 5 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s System Check Failed" % (name_tag, instance['InstanceId']),
            AlarmDescription='Status Check Failed (System) for 5 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='StatusCheckFailed_System',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=60,
            EvaluationPeriods=5,
            Threshold=1.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

        #Create Metric "Status Check Failed (Instance) for 20 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s Instance Check Failed" % (name_tag, instance['InstanceId']),
            AlarmDescription='Status Check Failed (Instance) for 20 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ec2_sns,
            ],
            MetricName='StatusCheckFailed_Instance',
            Namespace='AWS/EC2',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'InstanceId',
                    'Value': instance['InstanceId']
                },
            ],
            Period=60,
            EvaluationPeriods=20,
            Threshold=1.0,
            ComparisonOperator='GreaterThanOrEqualToThreshold'
        )

    for instance in instances:
        for dev in instance['BlockDeviceMappings']:
            if dev.get('Ebs', None) is None:
                continue
            vol_id = dev['Ebs']['VolumeId']
            print "Found EBS volume %s on instance %s" % (
                vol_id, instance['InstanceId'])

        #Create Metric "Volume Idle Time <= 30 sec (of 5 minutes) for 30 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High Volume Activity Warning" % (vol_id, instance['InstanceId']),
            AlarmDescription='Volume Idle Time <= 30 sec (of 5 minutes) for 30 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ebs_sns,
            ],
            MetricName='VolumeIdleTime',
            Namespace='AWS/EBS',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'VolumeId',
                    'Value': vol_id
                },
            ],
            Period=300,
            EvaluationPeriods=6,
            Threshold=30.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

        #Create Metric "Volume Idle Time <= 30 sec (of 5 minutes) for 60 Minutes"
        response = cw.put_metric_alarm(
            AlarmName="%s %s High Volume Activity Critical" % (vol_id, instance['InstanceId']),
            AlarmDescription='Volume Idle Time <= 30 sec (of 5 minutes) for 60 Minutes',
            ActionsEnabled=True,
            AlarmActions=[
                ebs_sns,
            ],
            MetricName='VolumeIdleTime',
            Namespace='AWS/EBS',
            Statistic='Average',
            Dimensions=[
                {
                    'Name': 'VolumeId',
                    'Value': vol_id
                },
            ],
            Period=300,
            EvaluationPeriods=12,
            Threshold=30.0,
            ComparisonOperator='LessThanOrEqualToThreshold'
        )

    database_instances = rd.describe_db_instances().get('DBInstances', [])
    for database in database_instances:
        print "Found RDS database %s" % (
            database['DBInstanceIdentifier'])
<|endoftext|>


<commit_before># The MIT License (MIT)
# Copyright (c) 2016 satojkovic

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import tensorflow as tf
import numpy as np
from scipy.misc import imresize
import common
import model
import argparse
import os
import preprocess
from concurrent.futures import ProcessPoolExecutor, as_completed
import util
import skimage.draw
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

def parse_cmdline():
    parser = argparse.ArgumentParser()
    parser.add_argument('img_fn', help='image filename')
    return parser.parse_args()

def logo_recognition(sess, img, obj_proposal, graph_params):
    # recognition results
    recog_results = {}
    recog_results['obj_proposal'] = obj_proposal

    # Resize image
    if img.shape != common.CNN_SHAPE:
        img = imresize(img, common.CNN_SHAPE, interp='bicubic')

    # Pre-processing
    img = preprocess.scaling(img)
    img = img.reshape((1, common.CNN_IN_HEIGHT, common.CNN_IN_WIDTH,
                       common.CNN_IN_CH)).astype(np.float32)

    # Logo recognition
    pred = sess.run(
        [graph_params['pred']], feed_dict={graph_params['target_image']: img})
    recog_results['pred_class'] = common.CLASS_NAME[np.argmax(pred)]
    recog_results['pred_prob'] = np.max(pred)

    return recog_results

def setup_graph():
    graph_params = {}
    graph_params['graph'] = tf.Graph()
    with graph_params['graph'].as_default():
        model_params = model.params()
        graph_params['target_image'] = tf.placeholder(
            tf.float32,
            shape=(1, common.CNN_IN_HEIGHT, common.CNN_IN_WIDTH,
                   common.CNN_IN_CH))
        logits = model.cnn(
            graph_params['target_image'], model_params, keep_prob=1.0)
        graph_params['pred'] = tf.nn.softmax(logits)
        graph_params['saver'] = tf.train.Saver()
    return graph_params

def main():
    args = parse_cmdline()
    img_fn = os.path.abspath(args.img_fn)
    if not os.path.exists(img_fn):
        print('Not found: {}'.format(img_fn))
        sys.exit(-1)
    else:
        print('Target image: {}'.format(img_fn))

    # Loaa target image
    target_image = util.load_target_image(img_fn)

    # Get object proposals
    object_proposals = util.get_object_proposals(target_image)

    # Setup computation graph
    graph_params = setup_graph()

    # Model initialize
    sess = tf.Session(graph=graph_params['graph'])
    tf.global_variables_initializer()
    if os.path.exists('models'):
        save_path = os.path.join('models', 'deep_logo_model')
        graph_params['saver'].restore(sess, save_path)
        print('Model restored')
    else:
        print('Initialized')

    # Logo recognition
    results = []
    for obj_proposal in object_proposals:
        x, y, w, h = obj_proposal
        crop_image = target_image[y:y + h, x:x + w]
        results.append(
            logo_recognition(sess, crop_image, obj_proposal, graph_params))

    del_idx = []
    for i, result in enumerate(results):
        if result['pred_class'] == common.CLASS_NAME[-1]:
            del_idx.append(i)
    results = np.delete(results, del_idx)

    # Non-max suppression
    nms_results = util.nms(results, pred_prob_th=0.999, iou_th=0.4)

    # Draw rectangles on the target image
    fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 6))
    ax.imshow(target_image)

    for result in nms_results:
        print(result)
        (x, y, w, h) = result['obj_proposal']
        ax.text(
            x,
            y,
            result['pred_class'],
            fontsize=13,
            bbox=dict(facecolor='red', alpha=0.7))
        rect = mpatches.Rectangle(
            (x, y), w, h, fill=False, edgecolor='red', linewidth=1)
        ax.add_patch(rect)
    plt.show()

if __name__ == '__main__':
    main()
<commit_msg>Paramter tuning<commit_after># The MIT License (MIT)
# Copyright (c) 2016 satojkovic

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import tensorflow as tf
import numpy as np
from scipy.misc import imresize
import common
import model
import argparse
import os
import preprocess
from concurrent.futures import ProcessPoolExecutor, as_completed
import util
import skimage.draw
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

def parse_cmdline():
    parser = argparse.ArgumentParser()
    parser.add_argument('img_fn', help='image filename')
    return parser.parse_args()

def logo_recognition(sess, img, obj_proposal, graph_params):
    # recognition results
    recog_results = {}
    recog_results['obj_proposal'] = obj_proposal

    # Resize image
    if img.shape != common.CNN_SHAPE:
        img = imresize(img, common.CNN_SHAPE, interp='bicubic')

    # Pre-processing
    img = preprocess.scaling(img)
    img = img.reshape((1, common.CNN_IN_HEIGHT, common.CNN_IN_WIDTH,
                       common.CNN_IN_CH)).astype(np.float32)

    # Logo recognition
    pred = sess.run(
        [graph_params['pred']], feed_dict={graph_params['target_image']: img})
    recog_results['pred_class'] = common.CLASS_NAME[np.argmax(pred)]
    recog_results['pred_prob'] = np.max(pred)

    return recog_results

def setup_graph():
    graph_params = {}
    graph_params['graph'] = tf.Graph()
    with graph_params['graph'].as_default():
        model_params = model.params()
        graph_params['target_image'] = tf.placeholder(
            tf.float32,
            shape=(1, common.CNN_IN_HEIGHT, common.CNN_IN_WIDTH,
                   common.CNN_IN_CH))
        logits = model.cnn(
            graph_params['target_image'], model_params, keep_prob=1.0)
        graph_params['pred'] = tf.nn.softmax(logits)
        graph_params['saver'] = tf.train.Saver()
    return graph_params

def main():
    args = parse_cmdline()
    img_fn = os.path.abspath(args.img_fn)
    if not os.path.exists(img_fn):
        print('Not found: {}'.format(img_fn))
        sys.exit(-1)
    else:
        print('Target image: {}'.format(img_fn))

    # Loaa target image
    target_image = util.load_target_image(img_fn)

    # Get object proposals
    object_proposals = util.get_object_proposals(target_image)

    # Setup computation graph
    graph_params = setup_graph()

    # Model initialize
    sess = tf.Session(graph=graph_params['graph'])
    tf.global_variables_initializer()
    if os.path.exists('models'):
        save_path = os.path.join('models', 'deep_logo_model')
        graph_params['saver'].restore(sess, save_path)
        print('Model restored')
    else:
        print('Initialized')

    # Logo recognition
    results = []
    for obj_proposal in object_proposals:
        x, y, w, h = obj_proposal
        crop_image = target_image[y:y + h, x:x + w]
        results.append(
            logo_recognition(sess, crop_image, obj_proposal, graph_params))

    del_idx = []
    for i, result in enumerate(results):
        if result['pred_class'] == common.CLASS_NAME[-1]:
            del_idx.append(i)
    results = np.delete(results, del_idx)

    # Non-max suppression
    nms_results = util.nms(results, pred_prob_th=0.999999, iou_th=0.4)

    # Draw rectangles on the target image
    fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 6))
    ax.imshow(target_image)

    for result in nms_results:
        print(result)
        (x, y, w, h) = result['obj_proposal']
        ax.text(
            x,
            y,
            result['pred_class'],
            fontsize=13,
            bbox=dict(facecolor='red', alpha=0.7))
        rect = mpatches.Rectangle(
            (x, y), w, h, fill=False, edgecolor='red', linewidth=1)
        ax.add_patch(rect)
    plt.show()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import ystockquote
import datetime
from dateutil.relativedelta import relativedelta
import json
import urllib2

def get_stock_year_price(symbol,year_begin, year_end):
	'''
	Get the prices of Stock {symbol} from year_begin (int) to year_end (int)

	Returns a dictionary like:
	{"Symbol": symbol, "Prices": {"2015-1-1":..., "2015-2-1":...}}

	'''
	d = datetime.datetime(year_begin,1,1)
	end_date = datetime.datetime(year_end,1,1)
	ret = {"Symbol":{},"Prices":{}}
	# iterating over month
	while  d < end_date:
		next_mon = d + relativedelta(months=1)
		# fetch this month's all trade days' data
		try:
			m =  ystockquote.get_historical_prices(symbol, d.strftime("%Y-%m-%d"), next_mon.strftime("%Y-%m-%d"))
		except urllib2.HTTPError:
			d = next_mon
			continue
		for day, data in m.iteritems():
			ret["Prices"][day] = data
		d = next_mon
	return ret<commit_msg>added symbol in returning json.<commit_after>#!/usr/bin/env python2
import ystockquote
import datetime
from dateutil.relativedelta import relativedelta
import json
import urllib2

def get_stock_year_price(symbol,year_begin, year_end):
	'''
	Get the prices of Stock {symbol} from year_begin (int) to year_end (int)

	Returns a dictionary like:
	{"Symbol": symbol, "Prices": {"2015-1-1":..., "2015-2-1":...}}

	'''
	d = datetime.datetime(year_begin,1,1)
	end_date = datetime.datetime(year_end,1,1)
	ret = {"Symbol":symbol,"Prices":{}}
	# iterating over month
	while  d < end_date:
		next_mon = d + relativedelta(months=1)
		# fetch this month's all trade days' data
		try:
			m =  ystockquote.get_historical_prices(symbol, d.strftime("%Y-%m-%d"), next_mon.strftime("%Y-%m-%d"))
		except urllib2.HTTPError:
			d = next_mon
			continue
		for day, data in m.iteritems():
			ret["Prices"][day] = data
		d = next_mon
	return ret<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import csv
import json
import base64
import common
import getpass
import hashlib
import urllib2

cols = ["date", "desc", "amount", "center"]

def downloadaccount(params):
    params["lastcheck"] = common.parsedate(params.get("lastcheck", "2000-01-01"))
    params.setdefault("seenids", [])
    params.setdefault("name", "Cash")
    if params.get("username") and not params.get("password"):
        params["password"] = getpass.getpass("Password for %s at %s: " % (params["username"], params["url"]))
    request = urllib2.Request(params["url"])
    if params.get("username") and params.get("password"):
        request.add_header("Authorization", "Basic %s" % base64.encodestring('%s:%s' % (params["username"], params["password"])).replace('\n', ''))
    f = urllib2.urlopen(request)
    transactions = []
    for entry in csv.reader(f):
        trans = {}
        for i in range(len(cols)):
            trans[cols[i]] = entry[i]
        if common.parsedate(trans["date"]) < params["lastcheck"]:
            continue
        trans["id"] = "%s-%s-%s" % (trans["date"], params["name"], hashlib.sha1(str(entry)).hexdigest())
        trans["account"] = params["name"]
        if trans["id"] in params["seenids"]:
            continue
    return {"transactions": transactions}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit(1)
    data = downloadaccount({"url": sys.argv[1]})
    json.dump(data, open("csvurl.json", "w"), indent=2, default=str)
<commit_msg>Actually save the transactions.<commit_after>#!/usr/bin/python
import sys
import csv
import json
import base64
import common
import getpass
import hashlib
import urllib2

cols = ["date", "desc", "amount", "center"]

def downloadaccount(params):
    params["lastcheck"] = common.parsedate(params.get("lastcheck", "2000-01-01"))
    params.setdefault("seenids", [])
    params.setdefault("name", "Cash")
    if params.get("username") and not params.get("password"):
        params["password"] = getpass.getpass("Password for %s at %s: " % (params["username"], params["url"]))
    request = urllib2.Request(params["url"])
    if params.get("username") and params.get("password"):
        request.add_header("Authorization", "Basic %s" % base64.encodestring('%s:%s' % (params["username"], params["password"])).replace('\n', ''))
    f = urllib2.urlopen(request)
    transactions = []
    for entry in csv.reader(f):
        trans = {}
        for i in range(len(cols)):
            trans[cols[i]] = entry[i]
        if common.parsedate(trans["date"]) < params["lastcheck"]:
            continue
        trans["id"] = "%s-%s-%s" % (trans["date"], params["name"], hashlib.sha1(str(entry)).hexdigest())
        trans["account"] = params["name"]
        if trans["id"] in params["seenids"]:
            continue
        transactions.append(trans)
    return {"transactions": transactions}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit(1)
    data = downloadaccount({"url": sys.argv[1]})
    json.dump(data, open("csvurl.json", "w"), indent=2, default=str)
<|endoftext|>


<commit_before>from bottle import run,route,abort,static_file,SimpleTemplate,default_app,app,request
from hashlib import md5
from pkg_resources import Requirement, resource_string
from setuptools.package_index import PackageIndex, egg_info_for_url as egg_info
from sh import git, hg, cd, cp
from subprocess import call
from yaml import load, dump
import logging.config
import os
import shutil
import sys
import tempfile
import os.path as opath

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

def lookup_config_file(filename):
    try :
        return next(filter(lambda f: opath.exists(f),
                [opath.join(opath.expanduser('~'), filename),
                 opath.join(opath.dirname(opath.abspath(sys.argv[0])),
                              '../etc/{}'.format(filename)),
                 '/etc/{}'.format(filename) ]))
    except StopIteration :
        return None

class Generic: pass
def get_config(data) :
    if isinstance(data, dict) :
        config = Generic()
        for key, value in data.items() :
            setattr(config, key, get_config(value))
        return config
    elif isinstance(data, (list, tuple)) :
        return [get_config(item) for item in data]
    return data

def get_app():
    app = default_app()
    default_config = {
        "nestegg_dir" : "{}/.nestegg".format(opath.expanduser('~')),
        "index_url": "https://pypi.python.org/simple",
        "port": 7654,  "fresh": "0",
    }
    ini_file = lookup_config_file("nestegg.ini")
    if ini_file :
        logging.config.fileConfig(ini_file)
    config_file = lookup_config_file("nestegg.yml")
    if config_file :
        with open(config_file,"r") as in_config :
            config = get_config(load(in_config))
        for key, value in default_config.items() :
            if not hasattr(config.nestegg,key):
                setattr(config.nestegg,key,value)
    else :
        config = get_config(default_config)
    neconfig = config.nestegg
    neconfig.pypi_dir = os.path.join(neconfig.nestegg_dir,"pypi")
    neconfig.checkout_dir = os.path.join(neconfig.nestegg_dir,"checkout")
    neconfig.source_dir = os.path.join(neconfig.nestegg_dir,"source_builds")
    neconfig.private_packages = set()
    os.makedirs(neconfig.pypi_dir,0o755, exist_ok=True)
    app.config['config'] = config
    return app

class NesteggPackageIndex(PackageIndex):
    def __init__(self, fresh, *args, **kwargs):
        self.fresh = "fresh" == "1"
        super().__init__(*args,**kwargs)

    def process_index(self, url, page):
        if self.fresh:
            del self.fetched_urls[url]
        super().process_index(url, page)

def source_build_dir(config, pkg_name) :
    return opath.join(config.nestegg.source_dir, pkg_name)

def source_build_file(config, pkg_name, file_name) :
    return opath.join(config.nestegg.source_dir, pkg_name, file_name)

def check_source_builds(config):
    cwd = os.getcwd()
    os.makedirs(config.nestegg.checkout_dir,exist_ok=True)
    for pkg in  config.nestegg.source_builds :
        if not hasattr(pkg, "private") : setattr(pkg, "private", False)
        elif pkg.private : config.nestegg.private_packages.add(pkg.name)
        if any(not opath.exists(source_build_file(config,pkg.name, v.dist_file))
                    for v in pkg.versions) :
            os.makedirs(source_build_dir(config,pkg.name), exist_ok=True)
            pkg_co_dir=opath.join(config.nestegg.checkout_dir,pkg.name)
            if opath.exists(pkg_co_dir): shutil.rmtree(pkg_co_dir)
            cd(config.nestegg.checkout_dir)
            call([pkg.repo_type, "clone", pkg.repo_url, pkg.name])
            cd(pkg_co_dir)
            for ver in pkg.versions :
                call([pkg.repo_type, "checkout", ver.tag])
                call(["python", "setup.py", "sdist"])
                cp(opath.join(pkg_co_dir,"dist",ver.dist_file),
                   source_build_dir(config, pkg.name))
    cd(cwd)

def file_md5(path) :
    m = md5()
    with open(path,"rb") as infile :
        for chunk in iter(lambda: infile.read(8192), b'') :
            m.update(chunk)
    return m.hexdigest()

def get_pkg_html(config, pkg_path, pkg_idx, pkg_name, pindex):
    os.makedirs(pkg_path, exist_ok=True)
    html_file = opath.join(pkg_path, "index.html")
    versions = [] if pkg_name in config.nestegg.private_packages or \
                     not pkg_idx[pkg_name] \
               else [egg_info(d.location) for d in pkg_idx[pkg_name]]
    if pkg_idx.fresh or not opath.exists(html_file) :
        source_pkg_dir = source_build_dir(config, pkg_name)
        if opath.exists(source_pkg_dir) :
            for fname in os.listdir(source_pkg_dir) :
                fpath = opath.join(source_pkg_dir,fname)
                if opath.isfile(fpath) :
                    cp(fpath, pkg_path)
                    versions.append((fname, "md5={}".format(file_md5(fpath))))
        info = {"name" : pkg_name,"versions": versions }
        with open(html_file, "w") as outfile :
            outfile.write(pindex.render(**info))
    return (pkg_path, "index.html")

@route('/simple/')
def get_root():
    config, pkg_idx = app.config['config'], app.config['pkg_idx']
    if not opath.exists(opath.join(config.nestegg.pypi_dir,"index.html")):
        log.debug("Updating base index")
        pkg_idx.scan_all()
        with open(opath.join(config.nestegg.pypi_dir,"index.html"), "w") as h :
            h.write(app.config['views'].gindex.render(
                pkgs=sorted(pkg_idx.package_pages)))
    return static_file("index.html", root=config.nestegg.pypi_dir)

def is_valid_package(config, pkg_idx, pkg_name) :
    if pkg_name in config.nestegg.private_packages : return True
    if  opath.exists(source_build_dir(config, pkg_name)) :
        if not pkg_idx[pkg_name] :
            config.nestegg.private_packages.add(pkg_name)
        return True
    return True if pkg_idx[pkg_name] else False

@route('/simple/<pkg_name>/')
def get_package(pkg_name):
    config, pkg_idx = request.app.config['config'], request.app.config['pkg_idx']
    if pkg_name not in config.nestegg.private_packages :
        pkg_idx.find_packages(Requirement.parse(pkg_name))
    pkg_path = opath.join(config.nestegg.pypi_dir, pkg_name)
    if is_valid_package(config, pkg_idx, pkg_name) :
        root, html = get_pkg_html(config, pkg_path, pkg_idx, pkg_name,
                request.app.config['views'].pindex)
        return static_file(html, root=root)
    else:
        abort(404, "No such package {}".format(pkg_name))

@route('/simple/<pkg_name>/<egg_name>')
@route('/simple/<pkg_name>/<egg_name>/')
def get_egg(pkg_name, egg_name):
    config, pkg_idx = request.app.config['config'], request.app.config['pkg_idx']
    log.debug("Package: {} egg:{}".format(pkg_name, egg_name))
    pkg_dir = opath.join(config.nestegg.pypi_dir, pkg_name)
    if not egg_name.startswith(pkg_name) :
        egg_name="{}-{}.".format(pkg_name,egg_name)
        if opath.exists(pkg_dir) :
            for fname in os.listdir(pkg_dir) :
                if fname.startswith(egg_name) and fname != egg_name :
                    egg_name = fname
                    break
    fpath = opath.join(config.nestegg.pypi_dir, pkg_name, egg_name)
    if not opath.exists(fpath) :
        pkg_idx.find_packages(Requirement.parse(pkg_name))
        for dist in pkg_idx[pkg_name] :
            if egg_info(dist.location)[0] == egg_name:
                log.debug("Fetch {}/{}".format(pkg_name,egg_name))
                tmp = tempfile.gettempdir()
                try :
                    shutil.move(pkg_idx.download(dist.location, tmp), fpath)
                    return static_file(egg_name, root=pkg_dir)
                except Exception as _e :
                    pass
        abort(404,"No egg found for {} {}".format(pkg_name,egg_name))
    else :
        return static_file(egg_name, root=pkg_dir)

def main() :
    app = get_app()
    config = app.config['config']
    pkg_idx = NesteggPackageIndex(config.nestegg.fresh, config.nestegg.index_url)
    app.config['pkg_idx'] = pkg_idx
    check_source_builds(app.config["config"])
    app.config['views'] = Generic()
    app.config['views'].pindex = \
        SimpleTemplate(resource_string('nestegg','views/package.tpl'))
    app.config['views'].gindex = \
        SimpleTemplate(resource_string('nestegg','views/all_packages.tpl'))
    run(app=app, host='0.0.0.0', port = config.nestegg.port)

if __name__ == "__main__" :
    main()
<commit_msg>Added archives capability<commit_after>from bottle import run,route,abort,static_file,SimpleTemplate,default_app,app,request
from hashlib import md5
from pkg_resources import Requirement, resource_string
from setuptools.package_index import PackageIndex, egg_info_for_url as egg_info
from sh import git, hg, cd, cp
from subprocess import call
from yaml import load, dump
import logging.config
import os
import shutil
import sys
import tempfile
import os.path as opath

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

def lookup_config_file(filename):
    try :
        return next(filter(lambda f: opath.exists(f),
                [opath.join(opath.expanduser('~'), filename),
                 opath.join(opath.dirname(opath.abspath(sys.argv[0])),
                              '../etc/{}'.format(filename)),
                 '/etc/{}'.format(filename) ]))
    except StopIteration :
        return None

class Generic: pass
def get_config(data) :
    if isinstance(data, dict) :
        config = Generic()
        for key, value in data.items() :
            setattr(config, key, get_config(value))
        return config
    elif isinstance(data, (list, tuple)) :
        return [get_config(item) for item in data]
    return data

def get_app():
    app = default_app()
    default_config = {
        "nestegg_dir" : "{}/.nestegg".format(opath.expanduser('~')),
        "index_url": "https://pypi.python.org/simple",
        "port": 7654,  "fresh": "0",
    }
    ini_file = lookup_config_file("nestegg.ini")
    if ini_file :
        logging.config.fileConfig(ini_file)
    config_file = lookup_config_file("nestegg.yml")
    if config_file :
        with open(config_file,"r") as in_config :
            config = get_config(load(in_config))
        for key, value in default_config.items() :
            if not hasattr(config.nestegg,key):
                setattr(config.nestegg,key,value)
    else :
        config = get_config(default_config)
    neconfig = config.nestegg
    neconfig.pypi_dir = os.path.join(neconfig.nestegg_dir,"pypi")
    neconfig.checkout_dir = os.path.join(neconfig.nestegg_dir,"checkout")
    neconfig.source_dir = os.path.join(neconfig.nestegg_dir,"source_builds")
    neconfig.archives_dir =os.path.join(neconfig.nestegg_dir,"archived_builds")
    neconfig.private_packages = set()
    os.makedirs(neconfig.pypi_dir,0o755, exist_ok=True)
    app.config['config'] = config
    return app

class NesteggPackageIndex(PackageIndex):
    def __init__(self, fresh, *args, **kwargs):
        self.fresh = "fresh" == "1"
        super().__init__(*args,**kwargs)

    def process_index(self, url, page):
        if self.fresh:
            del self.fetched_urls[url]
        super().process_index(url, page)

def source_build_dir(config, pkg_name) :
    return opath.join(config.nestegg.source_dir, pkg_name)

def archives_dir(config, pkg_name) :
    return opath.join(config.nestegg.archives_dir, pkg_name)

def source_build_file(config, pkg_name, file_name) :
    return opath.join(config.nestegg.source_dir, pkg_name, file_name)

def check_source_builds(config):
    cwd = os.getcwd()
    os.makedirs(config.nestegg.checkout_dir,exist_ok=True)
    for pkg in  config.nestegg.source_builds :
        if not hasattr(pkg, "private") : setattr(pkg, "private", False)
        elif pkg.private : config.nestegg.private_packages.add(pkg.name)
        if any(not opath.exists(source_build_file(config,pkg.name, v.dist_file))
                    for v in pkg.versions) :
            os.makedirs(source_build_dir(config,pkg.name), exist_ok=True)
            pkg_co_dir=opath.join(config.nestegg.checkout_dir,pkg.name)
            if opath.exists(pkg_co_dir): shutil.rmtree(pkg_co_dir)
            cd(config.nestegg.checkout_dir)
            call([pkg.repo_type, "clone", pkg.repo_url, pkg.name])
            cd(pkg_co_dir)
            for ver in pkg.versions :
                pyexe = "python" if not hasattr(ver,"python") else ver.python
                print("Python is {}".format(pyexe))
                call([pkg.repo_type, "checkout", ver.tag])
                call([pyexe, "setup.py", "sdist"])
                cp(opath.join(pkg_co_dir,"dist",ver.dist_file),
                   opath.join(source_build_dir(config, pkg.name),
                       ver.dist_file))
    cd(cwd)

def file_md5(path) :
    m = md5()
    with open(path,"rb") as infile :
        for chunk in iter(lambda: infile.read(8192), b'') :
            m.update(chunk)
    return m.hexdigest()

def get_pkg_html(config, pkg_path, pkg_idx, pkg_name, pindex):
    os.makedirs(pkg_path, exist_ok=True)
    html_file = opath.join(pkg_path, "index.html")
    if pkg_name in config.nestegg.private_packages or not pkg_idx[pkg_name] :
      versions = {}
    else :
        versions = dict((egg_info(d.location) for d in pkg_idx[pkg_name]))
    if pkg_idx.fresh or not opath.exists(html_file) :
        archives_pkg_dir = archives_dir(config, pkg_name)
        if opath.exists(archives_pkg_dir) :
            for fname in os.listdir(archives_pkg_dir) :
                fpath = opath.join(archives_pkg_dir,fname)
                if opath.isfile(fpath) :
                    cp(fpath, pkg_path)
                    versions[fname] =  "md5={}".format(file_md5(fpath))
        source_pkg_dir = source_build_dir(config, pkg_name)
        if opath.exists(source_pkg_dir) :
            for fname in os.listdir(source_pkg_dir) :
                fpath = opath.join(source_pkg_dir,fname)
                if opath.isfile(fpath) :
                    cp(fpath, pkg_path)
                    versions[fname] =  "md5={}".format(file_md5(fpath))
        info = {"name" : pkg_name,"versions": versions }
        with open(html_file, "w") as outfile :
            outfile.write(pindex.render(**info))
    return (pkg_path, "index.html")

@route('/simple/')
def get_root():
    config, pkg_idx = request.app.config['config'], request.app.config['pkg_idx']
    if not opath.exists(opath.join(config.nestegg.pypi_dir,"index.html")):
        log.debug("Updating base index")
        pkg_idx.scan_all()
        with open(opath.join(config.nestegg.pypi_dir,"index.html"), "w") as h :
            h.write(app.config['views'].gindex.render(
                pkgs=sorted(pkg_idx.package_pages)))
    return static_file("index.html", root=config.nestegg.pypi_dir)

def is_valid_package(config, pkg_idx, pkg_name) :
    if pkg_name in config.nestegg.private_packages : return True
    if opath.exists(archives_dir(config, pkg_name)) : return True
    if opath.exists(source_build_dir(config, pkg_name)) :
        if not pkg_idx[pkg_name] :
            config.nestegg.private_packages.add(pkg_name)
        return True
    return True if pkg_idx[pkg_name] else False

@route('/simple/<pkg_name>/')
def get_package(pkg_name):
    config, pkg_idx = request.app.config['config'], request.app.config['pkg_idx']
    if pkg_name not in config.nestegg.private_packages :
        pkg_idx.find_packages(Requirement.parse(pkg_name))
    pkg_path = opath.join(config.nestegg.pypi_dir, pkg_name)
    if is_valid_package(config, pkg_idx, pkg_name) :
        root, html = get_pkg_html(config, pkg_path, pkg_idx, pkg_name,
                request.app.config['views'].pindex)
        return static_file(html, root=root)
    else:
        abort(404, "No such package {}".format(pkg_name))

@route('/simple/<pkg_name>/<egg_name>')
@route('/simple/<pkg_name>/<egg_name>/')
def get_egg(pkg_name, egg_name):
    config, pkg_idx = request.app.config['config'], request.app.config['pkg_idx']
    log.debug("Package: {} egg:{}".format(pkg_name, egg_name))
    pkg_dir = opath.join(config.nestegg.pypi_dir, pkg_name)
    if not egg_name.startswith(pkg_name) :
        egg_name="{}-{}.".format(pkg_name,egg_name)
        if opath.exists(pkg_dir) :
            for fname in os.listdir(pkg_dir) :
                if fname.startswith(egg_name) and fname != egg_name :
                    egg_name = fname
                    break
    fpath = opath.join(config.nestegg.pypi_dir, pkg_name, egg_name)
    if not opath.exists(fpath) :
        pkg_idx.find_packages(Requirement.parse(pkg_name))
        for dist in pkg_idx[pkg_name] :
            if egg_info(dist.location)[0] == egg_name:
                log.debug("Fetch {}/{}".format(pkg_name,egg_name))
                tmp = tempfile.gettempdir()
                try :
                    shutil.move(pkg_idx.download(dist.location, tmp), fpath)
                    return static_file(egg_name, root=pkg_dir)
                except Exception as _e :
                    pass
        abort(404,"No egg found for {} {}".format(pkg_name,egg_name))
    else :
        return static_file(egg_name, root=pkg_dir)

def main() :
    app = get_app()
    config = app.config['config']
    pkg_idx = NesteggPackageIndex(config.nestegg.fresh, config.nestegg.index_url)
    app.config['pkg_idx'] = pkg_idx
    check_source_builds(app.config["config"])
    app.config['views'] = Generic()
    app.config['views'].pindex = \
        SimpleTemplate(resource_string('nestegg','views/package.tpl'))
    app.config['views'].gindex = \
        SimpleTemplate(resource_string('nestegg','views/all_packages.tpl'))
    run(app=app, host='0.0.0.0', port = config.nestegg.port)

if __name__ == "__main__" :
    main()
<|endoftext|>


<commit_before>import os
import logging
log = logging.getLogger(__file__)

from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import object_utils
from ott.utils import web_utils
from ott.utils.cache_base import CacheBase

class OsmCache(CacheBase):
    """ Does a 'smart' cache of a gtfs file
         1. it will look to see if a gtfs.zip file is in the cache, and download it and put it in the cache if not
         2. once cached, it will check to see that the file in the cache is the most up to date data...
    """
    pbf_url   = None
    pbf_name  = None
    pbf_path  = None
    pbf_url   = None

    meta_url  = None
    meta_name = None
    meta_path = None
    meta_url  = None

    osm_name  = None
    osm_path  = None

    top    = None
    bottom = None
    left   = None
    right  = None

    def __init__(self):
        ''' check osm cache
        '''
        super(OsmCache, self).__init__(section='osm')

        # step 1: cache dir management
        self.cache_expire = self.config.get_int('cache_expire', def_val=self.cache_expire)

        # step 2: urls
        self.pbf_url  = self.config.get('pbf_url')
        self.meta_url = self.config.get('meta_url')

        # step 3: file names
        name = self.config.get('name')
        self.pbf_name  = name + ".pbf"
        self.meta_name = name + ".html"
        self.osm_name  = name + ".osm"

        # step 4: file cache paths
        self.pbf_path  = os.path.join(self.cache_dir, self.pbf_name)
        self.meta_path = os.path.join(self.cache_dir, self.meta_name)
        self.osm_path  = os.path.join(self.cache_dir, self.osm_name)

        # step 5: get bbox from config
        self.top, self.bottom, self.left, self.right = self.config.get_bbox()

    def check_cached_osm(self, force_update=False):
        ''' if OSM .pbf file is out of date, download a new one.
            convert .pbf to .osm if .pbf file is newer than .osm file
        '''
        min_size = self.config.get_int('min_size', def_val=1000000)

        # step 1: download new osm pbf file if it's not new
        fresh = self.is_fresh_in_cache(self.pbf_path)
        sized = file_utils.is_min_sized(self.pbf_path, min_size)
        if force_update or not fresh or not sized:
            self.download_pbf()

        # step 2: .pbf to .osm
        if not file_utils.is_min_sized(self.pbf_path, min_size):
            log.warn("OSM PBF file {} is not big enough".format(self.pbf_path))
        else:
            fresh = self.is_fresh_in_cache(self.osm_path)
            sized = file_utils.is_min_sized(self.osm_path, min_size)
            pbf_newer = file_utils.is_a_newer_than_b(self.pbf_path, self.osm_path)
            if force_update or pbf_newer or not fresh or not sized:
                self.pbf_clip_bbox_to_osm()

        # step 3: .osm file check
        if not file_utils.is_min_sized(self.osm_path, min_size):
            e = "OSM file {} is not big enough".format(self.osm_path)
            raise Exception(e)

    def get_osmosis_exe(self):
        ''' get the path osmosis binary
            TODO - we should look for system installed osmosis first
        '''
        # step 1: get osmosis binary path (for ux or dos, ala c:\\ in path will get you a .bin extension)
        osmosis_dir = os.path.join(self.this_module_dir, "osmosis")
        osmosis_exe = os.path.join(osmosis_dir, "bin", "osmosis")
        if ":\\" in osmosis_exe:
            osmosis_exe = osmosis_exe + ".bat"

        # step 2: osmosis installed?
        if not os.path.exists(osmosis_exe):
            e = "OSMOSIS {} doesn't exist...\nMaybe cd into {} and run osmosis.sh".format(osmosis_exe, osmosis_dir)
            raise Exception(e)

        return osmosis_exe

    def pbf_clip_bbox_to_osm(self):
        ''' use osmosis to clip a bbox out of a .pbf, and output .osm file
            (file paths derrived by the cache paths & config)
        '''
        osmosis_exe = self.get_osmosis_exe()
        osmosis = "{} --rb {} --bounding-box top={} bottom={} left={} right={} completeWays=true --wx {}"
        osmosis_cmd = osmosis.format(osmosis_exe, self.pbf_path, self.top, self.bottom, self.left, self.right, self.osm_path)
        log.info(osmosis_cmd)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def osm_to_pbf(self, osm_path=None, pbf_path=None):
        ''' use osmosis to convert .osm file to .pbf
        '''
        if osm_path is None:
            osm_path = self.osm_path
        if pbf_path is None:
            pbf_path = osm_path + ".pbf"
        osmosis_exe = self.get_osmosis_exe()
        osmosis = '{} --read-xml {} --write-pbf {}'
        osmosis_cmd = osmosis.format(osmosis_exe, osm_path, pbf_path)
        log.info(osmosis_cmd)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def pbf_to_osm(self, osm_path=None, pbf_path=None):
        ''' use osmosis to convert .pbf to .osm file
        '''
        if osm_path is None:
            osm_path = self.osm_path
        if pbf_path is None:
            pbf_path = osm_path + ".pbf"
        osmosis_exe = self.get_osmosis_exe()
        osmosis = '{} --read-pbf {} --write-xml {}'
        osmosis_cmd = osmosis.format(osmosis_exe, pbf_path, osm_path)
        log.info(osmosis_cmd)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def download_pbf(self):
        log.info("wget {} to {}".format(self.pbf_url, self.pbf_path))
        file_utils.bkup(self.pbf_path)
        web_utils.wget(self.pbf_url, self.pbf_path)
        if self.meta_url:
            web_utils.wget(self.meta_url, self.meta_path)

    @classmethod
    def check_osm_file_against_cache(cls, app_dir, force_update=False):
        ''' check the .osm file in this cache against an osm file in another app's directory
        '''
        ret_val = False
        try:
            osm = OsmCache()
            app_osm_path = os.path.join(app_dir, osm.osm_name)
            refresh = file_utils.is_a_newer_than_b(osm.osm_path, app_osm_path)
            if refresh or force_update:
                log.info("cp {} to {}".format(osm.osm_name, app_dir))
                osm.cp_cached_file(osm.osm_name, app_dir)
                ret_val = True
        except Exception, e:
            log.warn(e)
        return ret_val

    @classmethod
    def load(cls):
        ''' run the SUM loader routines
        '''
        #import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.check_cached_osm(force_update=object_utils.is_force_update())

    @classmethod
    def convert_osm_to_pbf(cls):
        ''' run the SUM loader routines
        '''
        # import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.osm_to_pbf()

    @classmethod
    def convert_pbf_to_osm(cls):
        ''' run the SUM loader routines
        '''
        # import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.pbf_to_osm()
<commit_msg>convert fix ups ... file / path names<commit_after>import os
import re
import logging
log = logging.getLogger(__file__)

from ott.utils import exe_utils
from ott.utils import file_utils
from ott.utils import object_utils
from ott.utils import web_utils
from ott.utils.cache_base import CacheBase

class OsmCache(CacheBase):
    """ Does a 'smart' cache of a gtfs file
         1. it will look to see if a gtfs.zip file is in the cache, and download it and put it in the cache if not
         2. once cached, it will check to see that the file in the cache is the most up to date data...
    """
    pbf_url   = None
    pbf_name  = None
    pbf_path  = None
    pbf_url   = None

    meta_url  = None
    meta_name = None
    meta_path = None
    meta_url  = None

    osm_name  = None
    osm_path  = None

    top    = None
    bottom = None
    left   = None
    right  = None

    def __init__(self):
        ''' check osm cache
        '''
        super(OsmCache, self).__init__(section='osm')

        # step 1: cache dir management
        self.cache_expire = self.config.get_int('cache_expire', def_val=self.cache_expire)

        # step 2: urls
        self.pbf_url  = self.config.get('pbf_url')
        self.meta_url = self.config.get('meta_url')

        # step 3: file names
        name = self.config.get('name')
        self.pbf_name  = name + ".pbf"
        self.meta_name = name + ".html"
        self.osm_name  = name + ".osm"

        # step 4: file cache paths
        self.pbf_path  = os.path.join(self.cache_dir, self.pbf_name)
        self.meta_path = os.path.join(self.cache_dir, self.meta_name)
        self.osm_path  = os.path.join(self.cache_dir, self.osm_name)

        # step 5: get bbox from config
        self.top, self.bottom, self.left, self.right = self.config.get_bbox()

    def check_cached_osm(self, force_update=False):
        ''' if OSM .pbf file is out of date, download a new one.
            convert .pbf to .osm if .pbf file is newer than .osm file
        '''
        min_size = self.config.get_int('min_size', def_val=1000000)

        # step 1: download new osm pbf file if it's not new
        fresh = self.is_fresh_in_cache(self.pbf_path)
        sized = file_utils.is_min_sized(self.pbf_path, min_size)
        if force_update or not fresh or not sized:
            self.download_pbf()

        # step 2: .pbf to .osm
        if not file_utils.is_min_sized(self.pbf_path, min_size):
            log.warn("OSM PBF file {} is not big enough".format(self.pbf_path))
        else:
            fresh = self.is_fresh_in_cache(self.osm_path)
            sized = file_utils.is_min_sized(self.osm_path, min_size)
            pbf_newer = file_utils.is_a_newer_than_b(self.pbf_path, self.osm_path)
            if force_update or pbf_newer or not fresh or not sized:
                self.pbf_clip_bbox_to_osm()

        # step 3: .osm file check
        if not file_utils.is_min_sized(self.osm_path, min_size):
            e = "OSM file {} is not big enough".format(self.osm_path)
            raise Exception(e)

    def get_osmosis_exe(self):
        ''' get the path osmosis binary
            TODO - we should look for system installed osmosis first
        '''
        # step 1: get osmosis binary path (for ux or dos, ala c:\\ in path will get you a .bin extension)
        osmosis_dir = os.path.join(self.this_module_dir, "osmosis")
        osmosis_exe = os.path.join(osmosis_dir, "bin", "osmosis")
        if ":\\" in osmosis_exe:
            osmosis_exe = osmosis_exe + ".bat"

        # step 2: osmosis installed?
        if not os.path.exists(osmosis_exe):
            e = "OSMOSIS {} doesn't exist...\nMaybe cd into {} and run osmosis.sh".format(osmosis_exe, osmosis_dir)
            raise Exception(e)

        return osmosis_exe

    def pbf_clip_bbox_to_osm(self):
        ''' use osmosis to clip a bbox out of a .pbf, and output .osm file
            (file paths derrived by the cache paths & config)
        '''
        osmosis_exe = self.get_osmosis_exe()
        osmosis = "{} --rb {} --bounding-box top={} bottom={} left={} right={} completeWays=true --wx {}"
        osmosis_cmd = osmosis.format(osmosis_exe, self.pbf_path, self.top, self.bottom, self.left, self.right, self.osm_path)
        log.info(osmosis_cmd)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def osm_to_pbf(self, osm_path=None, pbf_path=None):
        ''' use osmosis to convert .osm file to .pbf
        '''
        if osm_path is None:
            osm_path = self.osm_path
        if pbf_path is None:
            pbf_path = re.sub('.osm$', '', osm_path) + ".pbf"
        osmosis_exe = self.get_osmosis_exe()
        osmosis = '{} --read-xml {} --write-pbf {}'
        osmosis_cmd = osmosis.format(osmosis_exe, osm_path, pbf_path)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def pbf_to_osm(self, osm_path=None, pbf_path=None):
        ''' use osmosis to convert .pbf to .osm file
        '''
        if osm_path is None:
            osm_path = self.osm_path
        if pbf_path is None:
            pbf_path = re.sub('.osm$', '', osm_path) + ".pbf"
        osmosis_exe = self.get_osmosis_exe()
        osmosis = '{} --read-pbf {} --write-xml {}'
        osmosis_cmd = osmosis.format(osmosis_exe, pbf_path, osm_path)
        exe_utils.run_cmd(osmosis_cmd, shell=True)

    def download_pbf(self):
        log.info("wget {} to {}".format(self.pbf_url, self.pbf_path))
        file_utils.bkup(self.pbf_path)
        web_utils.wget(self.pbf_url, self.pbf_path)
        if self.meta_url:
            web_utils.wget(self.meta_url, self.meta_path)

    @classmethod
    def check_osm_file_against_cache(cls, app_dir, force_update=False):
        ''' check the .osm file in this cache against an osm file in another app's directory
        '''
        ret_val = False
        try:
            osm = OsmCache()
            app_osm_path = os.path.join(app_dir, osm.osm_name)
            refresh = file_utils.is_a_newer_than_b(osm.osm_path, app_osm_path)
            if refresh or force_update:
                log.info("cp {} to {}".format(osm.osm_name, app_dir))
                osm.cp_cached_file(osm.osm_name, app_dir)
                ret_val = True
        except Exception, e:
            log.warn(e)
        return ret_val

    @classmethod
    def load(cls):
        ''' run the SUM loader routines
        '''
        #import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.check_cached_osm(force_update=object_utils.is_force_update())

    @classmethod
    def convert_osm_to_pbf(cls):
        ''' run the SUM loader routines
        '''
        # import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.osm_to_pbf()

    @classmethod
    def convert_pbf_to_osm(cls):
        ''' run the SUM loader routines
        '''
        # import pdb; pdb.set_trace()
        osm = OsmCache()
        osm.pbf_to_osm()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
open.py

Created by Thomas Mangin on 2009-11-05.
Copyright (c) 2009 Exa Networks. All rights reserved.
"""

from bgp.message.parent import *
from bgp.message.inet   import *

def new_Open (data):
	version = ord(data[0])
	if version != 4:
		# Only version 4 is supported nowdays..
		raise Notify(2,1,data[0])
	asn = unpack('!H',data[1:3])[0]
	hold_time = unpack('!H',data[3:5])[0]
	numeric = unpack('!L',data[5:9])[0]
	router_id = "%d.%d.%d.%d" % (numeric>>24,(numeric>>16)&0xFF,(numeric>>8)&0xFF,numeric&0xFF)
	capabilities = new_Capabilities(data[9:])
	return Open(version,asn,router_id,capabilities,hold_time)

# =================================================================== Open

class Open (Message):
	TYPE = chr(0x01)

	def __init__ (self,version,asn,router_id,capabilities,hold_time):
		self.version = Version(version)
		self.asn = ASN(asn)
		self.hold_time = HoldTime(hold_time)
		self.router_id = RouterID(router_id)
		self.capabilities = capabilities

	def message (self):
		return self._message("%s%s%s%s%s" % (self.version.pack(),self.asn.pack(),self.hold_time.pack(),self.router_id.pack(),chr(0)))

	def __str__ (self):
		return "OPEN version=%d asn=%d hold_time=%s router_id=%s capabilities=[%s]" % (self.version, self.asn, self.hold_time, self.router_id,self.capabilities)

# =================================================================== Version

class Version (int):
	def pack (self):
		return chr(self)

# =================================================================== HoldTime

class HoldTime (int):
	def pack (self):
		return pack('!H',self)

	def keepalive (self):
		return int(self/3)

	def __len__ (self):
		return 2

# =================================================================== RouterID

class RouterID (object):
	def __init__ (self,ip):
		self.ip = ip
		self.raw = socket.inet_pton(socket.AF_INET,ip)

	def afi (self):
		return AFI.ipv4

	def __len__ (self):
		return 4

	def pack (self):
		return self.raw

	def __str__ (self):
		return self.ip

	def __cmp__ (self,other):
		if type(self) == type(other):
			return self.raw == other.raw
		if self.raw == other:
			return cmp(self.raw,other)
		return cmp(self.ip,other)

# =================================================================== Graceful (Restart)
# RFC 4727

class Graceful (dict):
	TIME_MASK     = 0x0FFF
	FLAG_MASK     = 0xF000

	# 0x8 is binary 1000
	RESTART_STATE = 0x08
	FORWARDING_STATE = 0x80

	def __init__ (self,restart_flag,restart_time,protos):
		self.restart_flag = restart_flag
		self.restart_time = restart_time & Graceful.TIME_MASK
		for afi,safi,family_flag in protos:
			self[(afi,safi)] = family_flag & Graceful.FORWARDING_STATE

	def extract (self):
		print "............................"
		restart  = pack('!H',((self.restart_flag << 12) | (self.restart_time & Graful.TIME_MASK)))
		families = [(afi.pack(),safi.pack(),chr(self[(afi,safi)])) for (afi,safi) in self.keys()]
		sfamilies = ''.join(["%s%s%s" % (pafi,psafi,family) for (pafi,psafi,family) in families])
		return ["%s%s" % (restart,sfamilies)]

	def __str__ (self):
		families = [(str(afi),str(safi),hex(self[(afi,safi)])) for (afi,safi) in self.keys()]
		sfamilies = ' '.join(["%s/%s=%s" % (afi,safi,family) for (afi,safi,family) in families])
		return "Graceful Restart Flags %s Time %d %s" % (hex(self.restart_flag),self.restart_time,sfamilies)

# =================================================================== MultiProtocol

class MultiProtocol (list):
	def __str__ (self):
		return 'Multiprotocol ' + ' '.join(["%s %s" % (safi,ssafi) for (safi,ssafi) in [(str(afi),str(safi)) for (afi,safi) in self]])

	def extract (self):
		rs = []
		for v in self:
			rs.append(pack('!H',v[0]) + pack('!H',v[1]))
		return rs

# =================================================================== RouteRefresh

class RouteRefresh (list):
	def __str__ (self):
		return "Route Refresh (unparsed)"

	def extract (self):
		return []

class CiscoRouteRefresh (list):
	def __str__ (self):
		return "Cisco Route Refresh (unparsed)"

	def extract (self):
		return []

# =================================================================== Unknown

class Unknown (object):
	def __init__ (self,value):
		self.value = value

	def __str__ (self):
		if self.value in Capabilities.reserved: return 'Reserved %s' % str(self.value)
		if self.value in Capabilities.unassigned: return 'Unassigned %s' % str(self.value)
		return 'unknown %s' % str(self.value)

	def extract (self):
		return []

# =================================================================== Parameter

class Parameter (int):
	AUTHENTIFICATION_INFORMATION = 0x01 # Depreciated
	CAPABILITIES                 = 0x02

	def __str__ (self):
		if self == 0x01: return "AUTHENTIFICATION INFORMATION"
		if self == 0x02: return "OPTIONAL"
		return 'UNKNOWN'

# =================================================================== Capabilities

def _key_values (name,data):
	if len(data) < 2:
		raise Notify(2,0,"bad length for OPEN %s (<2)" % name)
	l = ord(data[1])
	boundary = l+2
	if len(data) < boundary:
		raise Notify(2,0,"bad length for OPEN %s (buffer underrun)" % name)
	key = ord(data[0])
	value = data[2:boundary]
	rest = data[boundary:]
	return key,value,rest

def new_Capabilities (data):
	capabilities = Capabilities()
	option_len = ord(data[0])
	if option_len:
		data = data[1:]
		while data:
			key,value,data = _key_values('parameter',data)
			# Paramaters must only be sent once.
			if key == Parameter.AUTHENTIFICATION_INFORMATION:
				raise Notify(2,5)
				continue

			if key == Parameter.CAPABILITIES:
				k,v,r = _key_values('capability',value)
				if r:
					raise Notify(2,0,"bad length for OPEN %s (size mismatch)" % 'capability')

				if k == Capabilities.MULTIPROTOCOL_EXTENSIONS:
					if k not in capabilities:
						capabilities[k] = MultiProtocol()
					afi = AFI(unpack('!H',value[2:4])[0])
					safi = SAFI(ord(value[5]))
					capabilities[k].append((afi,safi))
					continue

				if k == Capabilities.GRACEFUL_RESTART:
					restart = unpack('!H',value[2:4])[0]
					restart_flag = restart >> 12
					restart_time = restart & Graceful.TIME_MASK
					value = value[4:]
					families = []
					while value:
						afi = AFI(unpack('!H',value[:2])[0])
						safi = SAFI(ord(value[2]))
						flag_family = ord(value[0])
						families.append((afi,safi,flag_family))
						value = value[4:]
					capabilities[k] = Graceful(restart_flag,restart_time,families)
					continue

				if k == Capabilities.FOUR_BYTES_ASN:
					capabilities[k] = ASN(unpack('!L',value[2:6])[0]).four()
					continue

				if k == Capabilities.ROUTE_REFRESH:
					capabilities[k] = RouteRefresh()
					continue

				if k == Capabilities.CISCO_ROUTE_REFRESH:
					capabilities[k] = CiscoRouteRefresh()
					continue

				if k not in capabilities:
					capabilities[k] = Unknown(k)
				if value[2:]:
					capabilities[k].append([ord(_) for _ in value[2:]])
			else:
				raise Notify(2,0,'unknow OPEN parameter %s' % hex(key))
	return capabilities

# =================================================================== Capabilities
# http://www.iana.org/assignments/capability-codes/

class Capabilities (dict):
	RESERVED                 = 0x00 # [RFC5492]
	MULTIPROTOCOL_EXTENSIONS = 0x01 # [RFC2858]
	ROUTE_REFRESH            = 0x02 # [RFC2918]
	OUTBOUND_ROUTE_FILTERING = 0x03 # [RFC5291]
	MULTIPLE_ROUTES          = 0x04 # [RFC3107]
	EXTENDED_NEXT_HOP        = 0x05 # [RFC5549]
	#6-63      Unassigned
	GRACEFUL_RESTART         = 0x40 # [RFC4724]
	FOUR_BYTES_ASN           = 0x41 # [RFC4893]
	# 66 Deprecated
	DYNAMIC_CAPABILITY       = 0x43 # [Chen]
	MULTISESSION_BGP         = 0x44 # [Appanna]
	ADD_PATH                 = 0x45 # [draft-ietf-idr-add-paths]
	# 70-127    Unassigned
	CISCO_ROUTE_REFRESH      = 0x80 # I Can only find reference to this in the router logs
	# 128-255   Reserved for Private Use [RFC5492]

	unassigned = range(70,128)
	reserved = range(128,256)

	def announced (self,capability):
		return self.has_key(capability)

	def __str__ (self):
		r = []
		for key in self.keys():
			if key == self.MULTIPROTOCOL_EXTENSIONS:
				r += ['Multiprotocol for ' + ' '.join(["%s %s" % (str(afi),str(safi)) for (afi,safi) in self[key]])]
			elif key == self.ROUTE_REFRESH:
				r += ['Route Refresh']
			elif key == self.CISCO_ROUTE_REFRESH:
				r += ['Cisco Route Refresh']
			elif key == self.GRACEFUL_RESTART:
				r += ['Graceful Restart']
			elif key == self.FOUR_BYTES_ASN:
				r += ['4Bytes AS %d' % self[key]]
			elif key in self._reserved:
				r += ['private use capability %d' % key]
			elif key in self._unassigned:
				r += ['unassigned capability %d' % key]
			else:
				r+= ['unhandled capability %d' % key]
		return ', '.join(r)

	def default (self,restarted):
		families = ((AFI(AFI.ipv4),SAFI(SAFI.unicast)),(AFI(AFI.ipv6),SAFI(SAFI.unicast)))

		mp = MultiProtocol()
		mp.extend(families)
		self[Capabilities.MULTIPROTOCOL_EXTENSIONS] = mp

		# XXX: RFC 4727 Section 4.0, says that the time SHOULD be inferiour or equal to the HOLDTIME ...
		if restarted:
			self[Capabilities.GRACEFUL_RESTART] = Graceful(Graceful.RESTART_STATE,120,[(afi,safi,Graceful.FORWARDING_STATE) for (afi,safi) in families])
		else:
			self[Capabilities.GRACEFUL_RESTART] = Graceful(0x0,120,[(afi,safi,Graceful.FORWARDING_STATE) for (afi,safi) in families])

		return self

	def pack (self):
		rs = []
		for k,capabilities in self.iteritems():
			for capability in capabilities.extract():
				rs.append("%s%s%s" % (chr(k),chr(len(capability)),capability))
		return "".join(["%s%s%s" % (chr(2),chr(len(r)),r) for r in rs])

	def __str__ (self):
		r = []
		for k in self.keys():
			r.append(str(self[k]))
		return ', '.join(r)
<commit_msg>actually sending the capabilities in the open packet does help a lot<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
open.py

Created by Thomas Mangin on 2009-11-05.
Copyright (c) 2009 Exa Networks. All rights reserved.
"""

from bgp.message.parent import *
from bgp.message.inet   import *

def new_Open (data):
	version = ord(data[0])
	if version != 4:
		# Only version 4 is supported nowdays..
		raise Notify(2,1,data[0])
	asn = unpack('!H',data[1:3])[0]
	hold_time = unpack('!H',data[3:5])[0]
	numeric = unpack('!L',data[5:9])[0]
	router_id = "%d.%d.%d.%d" % (numeric>>24,(numeric>>16)&0xFF,(numeric>>8)&0xFF,numeric&0xFF)
	capabilities = new_Capabilities(data[9:])
	return Open(version,asn,router_id,capabilities,hold_time)

# =================================================================== Open

class Open (Message):
	TYPE = chr(0x01)

	def __init__ (self,version,asn,router_id,capabilities,hold_time):
		self.version = Version(version)
		self.asn = ASN(asn)
		self.hold_time = HoldTime(hold_time)
		self.router_id = RouterID(router_id)
		self.capabilities = capabilities

	def message (self):
		return self._message("%s%s%s%s%s" % (self.version.pack(),self.asn.pack(),self.hold_time.pack(),self.router_id.pack(),self.capabilities.pack()))

	def __str__ (self):
		return "OPEN version=%d asn=%d hold_time=%s router_id=%s capabilities=[%s]" % (self.version, self.asn, self.hold_time, self.router_id,self.capabilities)

# =================================================================== Version

class Version (int):
	def pack (self):
		return chr(self)

# =================================================================== HoldTime

class HoldTime (int):
	def pack (self):
		return pack('!H',self)

	def keepalive (self):
		return int(self/3)

	def __len__ (self):
		return 2

# =================================================================== RouterID

class RouterID (object):
	def __init__ (self,ip):
		self.ip = ip
		self.raw = socket.inet_pton(socket.AF_INET,ip)

	def afi (self):
		return AFI.ipv4

	def __len__ (self):
		return 4

	def pack (self):
		return self.raw

	def __str__ (self):
		return self.ip

	def __cmp__ (self,other):
		if type(self) == type(other):
			return self.raw == other.raw
		if self.raw == other:
			return cmp(self.raw,other)
		return cmp(self.ip,other)

# =================================================================== Graceful (Restart)
# RFC 4727

class Graceful (dict):
	TIME_MASK     = 0x0FFF
	FLAG_MASK     = 0xF000

	# 0x8 is binary 1000
	RESTART_STATE = 0x08
	FORWARDING_STATE = 0x80

	def __init__ (self,restart_flag,restart_time,protos):
		self.restart_flag = restart_flag
		self.restart_time = restart_time & Graceful.TIME_MASK
		for afi,safi,family_flag in protos:
			self[(afi,safi)] = family_flag & Graceful.FORWARDING_STATE

	def extract (self):
		restart  = pack('!H',((self.restart_flag << 12) | (self.restart_time & Graceful.TIME_MASK)))
		families = [(afi.pack(),safi.pack(),chr(self[(afi,safi)])) for (afi,safi) in self.keys()]
		sfamilies = ''.join(["%s%s%s" % (pafi,psafi,family) for (pafi,psafi,family) in families])
		return ["%s%s" % (restart,sfamilies)]

	def __str__ (self):
		families = [(str(afi),str(safi),hex(self[(afi,safi)])) for (afi,safi) in self.keys()]
		sfamilies = ' '.join(["%s/%s=%s" % (afi,safi,family) for (afi,safi,family) in families])
		return "Graceful Restart Flags %s Time %d %s" % (hex(self.restart_flag),self.restart_time,sfamilies)

# =================================================================== MultiProtocol

class MultiProtocol (list):
	def __str__ (self):
		return 'Multiprotocol ' + ' '.join(["%s %s" % (safi,ssafi) for (safi,ssafi) in [(str(afi),str(safi)) for (afi,safi) in self]])

	def extract (self):
		rs = []
		for v in self:
			rs.append(pack('!H',v[0]) + pack('!H',v[1]))
		return rs

# =================================================================== RouteRefresh

class RouteRefresh (list):
	def __str__ (self):
		return "Route Refresh (unparsed)"

	def extract (self):
		return []

class CiscoRouteRefresh (list):
	def __str__ (self):
		return "Cisco Route Refresh (unparsed)"

	def extract (self):
		return []

# =================================================================== Unknown

class Unknown (object):
	def __init__ (self,value):
		self.value = value

	def __str__ (self):
		if self.value in Capabilities.reserved: return 'Reserved %s' % str(self.value)
		if self.value in Capabilities.unassigned: return 'Unassigned %s' % str(self.value)
		return 'unknown %s' % str(self.value)

	def extract (self):
		return []

# =================================================================== Parameter

class Parameter (int):
	AUTHENTIFICATION_INFORMATION = 0x01 # Depreciated
	CAPABILITIES                 = 0x02

	def __str__ (self):
		if self == 0x01: return "AUTHENTIFICATION INFORMATION"
		if self == 0x02: return "OPTIONAL"
		return 'UNKNOWN'

# =================================================================== Capabilities

def _key_values (name,data):
	if len(data) < 2:
		raise Notify(2,0,"bad length for OPEN %s (<2)" % name)
	l = ord(data[1])
	boundary = l+2
	if len(data) < boundary:
		raise Notify(2,0,"bad length for OPEN %s (buffer underrun)" % name)
	key = ord(data[0])
	value = data[2:boundary]
	rest = data[boundary:]
	return key,value,rest

def new_Capabilities (data):
	capabilities = Capabilities()
	option_len = ord(data[0])
	if option_len:
		data = data[1:]
		while data:
			key,value,data = _key_values('parameter',data)
			# Paramaters must only be sent once.
			if key == Parameter.AUTHENTIFICATION_INFORMATION:
				raise Notify(2,5)
				continue

			if key == Parameter.CAPABILITIES:
				k,v,r = _key_values('capability',value)
				if r:
					raise Notify(2,0,"bad length for OPEN %s (size mismatch)" % 'capability')

				if k == Capabilities.MULTIPROTOCOL_EXTENSIONS:
					if k not in capabilities:
						capabilities[k] = MultiProtocol()
					afi = AFI(unpack('!H',value[2:4])[0])
					safi = SAFI(ord(value[5]))
					capabilities[k].append((afi,safi))
					continue

				if k == Capabilities.GRACEFUL_RESTART:
					restart = unpack('!H',value[2:4])[0]
					restart_flag = restart >> 12
					restart_time = restart & Graceful.TIME_MASK
					value = value[4:]
					families = []
					while value:
						afi = AFI(unpack('!H',value[:2])[0])
						safi = SAFI(ord(value[2]))
						flag_family = ord(value[0])
						families.append((afi,safi,flag_family))
						value = value[4:]
					capabilities[k] = Graceful(restart_flag,restart_time,families)
					continue

				if k == Capabilities.FOUR_BYTES_ASN:
					capabilities[k] = ASN(unpack('!L',value[2:6])[0]).four()
					continue

				if k == Capabilities.ROUTE_REFRESH:
					capabilities[k] = RouteRefresh()
					continue

				if k == Capabilities.CISCO_ROUTE_REFRESH:
					capabilities[k] = CiscoRouteRefresh()
					continue

				if k not in capabilities:
					capabilities[k] = Unknown(k)
				if value[2:]:
					capabilities[k].append([ord(_) for _ in value[2:]])
			else:
				raise Notify(2,0,'unknow OPEN parameter %s' % hex(key))
	return capabilities

# =================================================================== Capabilities
# http://www.iana.org/assignments/capability-codes/

class Capabilities (dict):
	RESERVED                 = 0x00 # [RFC5492]
	MULTIPROTOCOL_EXTENSIONS = 0x01 # [RFC2858]
	ROUTE_REFRESH            = 0x02 # [RFC2918]
	OUTBOUND_ROUTE_FILTERING = 0x03 # [RFC5291]
	MULTIPLE_ROUTES          = 0x04 # [RFC3107]
	EXTENDED_NEXT_HOP        = 0x05 # [RFC5549]
	#6-63      Unassigned
	GRACEFUL_RESTART         = 0x40 # [RFC4724]
	FOUR_BYTES_ASN           = 0x41 # [RFC4893]
	# 66 Deprecated
	DYNAMIC_CAPABILITY       = 0x43 # [Chen]
	MULTISESSION_BGP         = 0x44 # [Appanna]
	ADD_PATH                 = 0x45 # [draft-ietf-idr-add-paths]
	# 70-127    Unassigned
	CISCO_ROUTE_REFRESH      = 0x80 # I Can only find reference to this in the router logs
	# 128-255   Reserved for Private Use [RFC5492]

	unassigned = range(70,128)
	reserved = range(128,256)

	def announced (self,capability):
		return self.has_key(capability)

	def __str__ (self):
		r = []
		for key in self.keys():
			if key == self.MULTIPROTOCOL_EXTENSIONS:
				r += ['Multiprotocol for ' + ' '.join(["%s %s" % (str(afi),str(safi)) for (afi,safi) in self[key]])]
			elif key == self.ROUTE_REFRESH:
				r += ['Route Refresh']
			elif key == self.CISCO_ROUTE_REFRESH:
				r += ['Cisco Route Refresh']
			elif key == self.GRACEFUL_RESTART:
				r += ['Graceful Restart']
			elif key == self.FOUR_BYTES_ASN:
				r += ['4Bytes AS %d' % self[key]]
			elif key in self._reserved:
				r += ['private use capability %d' % key]
			elif key in self._unassigned:
				r += ['unassigned capability %d' % key]
			else:
				r+= ['unhandled capability %d' % key]
		return ', '.join(r)

	def default (self,restarted):
		families = ((AFI(AFI.ipv4),SAFI(SAFI.unicast)),(AFI(AFI.ipv6),SAFI(SAFI.unicast)))

		mp = MultiProtocol()
		mp.extend(families)
		self[Capabilities.MULTIPROTOCOL_EXTENSIONS] = mp

		# XXX: RFC 4727 Section 4.0, says that the time SHOULD be inferiour or equal to the HOLDTIME ...
		if restarted:
			self[Capabilities.GRACEFUL_RESTART] = Graceful(Graceful.RESTART_STATE,120,[(afi,safi,Graceful.FORWARDING_STATE) for (afi,safi) in families])
		else:
			self[Capabilities.GRACEFUL_RESTART] = Graceful(0x0,120,[(afi,safi,Graceful.FORWARDING_STATE) for (afi,safi) in families])

		return self

	def pack (self):
		rs = []
		for k,capabilities in self.iteritems():
			for capability in capabilities.extract():
				rs.append("%s%s%s" % (chr(k),chr(len(capability)),capability))
		parameters = "".join(["%s%s%s" % (chr(2),chr(len(r)),r) for r in rs])
		return "%s%s" % (chr(len(parameters)),parameters)

	def __str__ (self):
		r = []
		for k in self.keys():
			r.append(str(self[k]))
		return ', '.join(r)
<|endoftext|>


<commit_before># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u

from streams.potential import *
from streams.simulation import Particle, TestParticleSimulation
from streams.data.gaia import parallax_error

def _variance_statistic(potential, xs, vs, sgr_cen):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        xs : ndarray
            An array of positions for all particles at all times
        vs : ndarray
            An array of velocities for all particles at all times
        sgr_cen : SgrCen
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    # Define tidal radius, escape velocity for satellite
    msat = 2.5E8 # M_sun
    sgr_orbital_radius = np.sqrt(sgr_cen.x**2 + sgr_cen.y**2 + sgr_cen.z**2)
    m_halo_enclosed = potential.params["v_halo"]**2 * sgr_orbital_radius/potential.params["_G"]
    mass_enclosed = potential.params["M_disk"] + potential.params["M_sph"] + m_halo_enclosed

    r_tides = sgr_orbital_radius * (msat / mass_enclosed)**(1./3)
    v_escs = np.sqrt(potential.params["_G"] * msat / r_tides)

    # N = number of particles
    N = xs.shape[1]

    # Distance to satellite center and total velocity
    d = np.sqrt((xs[:,:,0] - sgr_cen.x[:,np.newaxis].repeat(N, axis=1))**2 +
                (xs[:,:,1] - sgr_cen.y[:,np.newaxis].repeat(N, axis=1))**2 +
                (xs[:,:,2] - sgr_cen.z[:,np.newaxis].repeat(N, axis=1))**2) / r_tides[:,np.newaxis].repeat(N, axis=1)

    v = np.sqrt((vs[:,:,0] - sgr_cen.vx[:,np.newaxis].repeat(N, axis=1))**2 +
                (vs[:,:,1] - sgr_cen.vy[:,np.newaxis].repeat(N, axis=1))**2 +
                (vs[:,:,2] - sgr_cen.vz[:,np.newaxis].repeat(N, axis=1))**2) / v_escs[:,np.newaxis].repeat(N, axis=1)

    idx = np.argmin(d**2 + v**2, axis=0)

    min_ds = []
    for ii,jj in enumerate(idx):
        min_ds.append(d[jj,ii])

    min_vs = []
    for ii,jj in enumerate(idx):
        min_vs.append(v[jj,ii])

    return np.var(min_ds) + np.var(min_vs)

def back_integrate(potential, sgr_snap, sgr_cen, dt):
    """ Given the particle snapshot information and a potential, integrate the particles
        backwards and return the variance scalar.
    """

    # Initialize particle simulation with full potential
    simulation = TestParticleSimulation(potential=potential)

    for ii in range(sgr_snap.num):
        p = Particle(position=(sgr_snap.data["x"][ii], sgr_snap.data["y"][ii], sgr_snap.data["z"][ii]), # kpc
                     velocity=(sgr_snap.data["vx"][ii], sgr_snap.data["vy"][ii], sgr_snap.data["vz"][ii]), # kpc/Myr
                     mass=1.) # M_sol
        simulation.add_particle(p)

    ts, xs, vs = simulation.run(t1=max(sgr_cen.data["t"]), t2=min(sgr_cen.data["t"]), dt=-dt)

    return _variance_statistic(potential, xs, vs, sgr_cen)

def add_observational_uncertainty(x,y,z,vx,vy,vz):
    """ Given 3D galactocentric position and velocity, transform to heliocentric
        coordinates, apply observational uncertainty estimates, then transform
        back to galactocentric frame.
    """
    # Transform to heliocentric coordinates
    rsun = 8. # kpc

    x += rsun

    d = np.sqrt(x**2 + y**2 + z**2)
    vr = (x*vx + y*vy + z*vz) / d

    # proper motions in km/s/kpc
    rad = np.sqrt(x**2 + y**2)
    vrad = (x*vx + y*vy) / rad
    mul = (x*vy - y*vx) / rad / d
    mub = (-z*vrad + rad*vz) / d**2

    # angular position
    sinb = z/d
    cosb = rad/d
    cosl = x/rad
    sinl = y/rad

    # DISTANCE ERROR -- assuming 2% distances from RR Lyrae mid-IR
    d += np.random.normal(0., 0.02*d)

    # VELOCITY ERROR -- 5 km/s (TODO: ???)
    vr += np.random.normal(0., (5.*u.km/u.s).to(u.kpc/u.Myr).value)

    dmu = proper_motion_error(d)

    # translate to radians/year
    conv1 = np.pi/180./60./60./1.e6
    # translate to km/s from  kpc/year
    kmpkpc = 3.085678e16
    secperyr = 3.1536e7
    conv2 = kmpkpc/secperyr
    dmu = dmu*conv1*conv2

    mul += np.random.normal(0., dmu)
    mub += np.random.normal(0., dmu)

    # CONVERT BACK
    x = d*cosb*cosl - rsun
    y = d*cosb*sinl
    z = d*sinb

    vx = vr*cosb*cosl - d*mul*sinl - d*mub*sinb*cosl
    vy = vr*cosb*sinl + d*mul*cosl - d*mub*sinb*sinl
    vz = vr*sinb + d*mub*cosb

    return (x,y,z,vx,vy,vz)

def back_integrate_with_errors(potential, sgr_snap, sgr_cen, dt):
    """ Given the particle snapshot information and a potential, integrate the particles
        backwards WITH "realistic" observational uncertainties.
    """

    # Initialize particle simulation with full potential
    simulation = TestParticleSimulation(potential=potential)

    # Distances in kpc, velocities in kpc/Myr
    x, y, z = sgr_snap.data["x"], sgr_snap.data["y"], sgr_snap.data["z"]
    vx, vy, vz = sgr_snap.data["vx"], sgr_snap.data["vy"], sgr_snap.data["vz"]

    x,y,z,vx,vy,vz = add_observational_uncertainty(x,y,z,vx,vy,vz)

    for ii in range(sgr_snap.num):
        p = Particle(position=(x[ii], y[ii], z[ii]), # kpc
                     velocity=(vx[ii], vy[ii] ,vz[ii]), # kpc/Myr
                     mass=1.) # M_sol
        simulation.add_particle(p)

    ts, xs, vs = simulation.run(t1=max(sgr_cen.data["t"]), t2=min(sgr_cen.data["t"]), dt=-dt)

    return _variance_statistic(potential, xs, vs, sgr_cen)<commit_msg>make one function, with errors keyword<commit_after># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u

from streams.potential import *
from streams.simulation import Particle, TestParticleSimulation
from streams.data.gaia import parallax_error, rr_lyrae_M_V

def _variance_statistic(potential, xs, vs, sgr_cen):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        xs : ndarray
            An array of positions for all particles at all times
        vs : ndarray
            An array of velocities for all particles at all times
        sgr_cen : SgrCen
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    # Define tidal radius, escape velocity for satellite
    msat = 2.5E8 # M_sun
    sgr_orbital_radius = np.sqrt(sgr_cen.x**2 + sgr_cen.y**2 + sgr_cen.z**2)
    m_halo_enclosed = potential.params["v_halo"]**2 * sgr_orbital_radius/potential.params["_G"]
    mass_enclosed = potential.params["M_disk"] + potential.params["M_sph"] + m_halo_enclosed

    r_tides = sgr_orbital_radius * (msat / mass_enclosed)**(1./3)
    v_escs = np.sqrt(potential.params["_G"] * msat / r_tides)

    # N = number of particles
    N = xs.shape[1]

    # Distance to satellite center and total velocity
    d = np.sqrt((xs[:,:,0] - sgr_cen.x[:,np.newaxis].repeat(N, axis=1))**2 +
                (xs[:,:,1] - sgr_cen.y[:,np.newaxis].repeat(N, axis=1))**2 +
                (xs[:,:,2] - sgr_cen.z[:,np.newaxis].repeat(N, axis=1))**2) / r_tides[:,np.newaxis].repeat(N, axis=1)

    v = np.sqrt((vs[:,:,0] - sgr_cen.vx[:,np.newaxis].repeat(N, axis=1))**2 +
                (vs[:,:,1] - sgr_cen.vy[:,np.newaxis].repeat(N, axis=1))**2 +
                (vs[:,:,2] - sgr_cen.vz[:,np.newaxis].repeat(N, axis=1))**2) / v_escs[:,np.newaxis].repeat(N, axis=1)

    idx = np.argmin(d**2 + v**2, axis=0)

    min_ds = []
    for ii,jj in enumerate(idx):
        min_ds.append(d[jj,ii])

    min_vs = []
    for ii,jj in enumerate(idx):
        min_vs.append(v[jj,ii])

    return np.var(min_ds) + np.var(min_vs)

def back_integrate(potential, sgr_snap, sgr_cen, dt, errors=False):
    """ Given the particle snapshot information and a potential, integrate the particles
        backwards and return the variance scalar.
    """

    # Initialize particle simulation with full potential
    simulation = TestParticleSimulation(potential=potential)

    # Distances in kpc, velocities in kpc/Myr
    x, y, z = sgr_snap.data["x"]*u.kpc, sgr_snap.data["y"]*u.kpc, sgr_snap.data["z"]*u.kpc
    vx, vy, vz = sgr_snap.data["vx"]*u.kpc/u.Myr, sgr_snap.data["vy"]*u.kpc/u.Myr, sgr_snap.data["vz"]*u.kpc/u.Myr

    if errors:
        x,y,z,vx,vy,vz = add_observational_uncertainties(x,y,z,vx,vy,vz)

    for ii in range(sgr_snap.num):
        p = Particle(position=(x[ii].to(u.kpc).value, y[ii].to(u.kpc).value, z[ii].to(u.kpc).value), # kpc
                     velocity=(vx[ii].to(u.kpc/u.Myr).value, vy[ii].to(u.kpc/u.Myr).value, vz[ii].to(u.kpc/u.Myr).value), # kpc/Myr
                     mass=1.) # M_sol
        simulation.add_particle(p)

    ts, xs, vs = simulation.run(t1=max(sgr_cen.data["t"]), t2=min(sgr_cen.data["t"]), dt=-dt)

    return _variance_statistic(potential, xs, vs, sgr_cen)<|endoftext|>


<commit_before>from flask import abort, request, current_app
from functools import wraps
import authorise as auth
import logging

def authenticate(f):
    """
    Decorator to require api authentication

    Args:
        f: flask function
    Returns:
       function: decorated function or abort(401)
    """
    @wraps(f)
    def decorated(*args, **kwargs):

        #Load the authentication rule from configs, based on the request url_rule.
        logging.warning( "Requested url: {}".format(request.url_rule) )
        auth_rule = current_app.config['AUTH'].get(
            str(request.url_rule),
            current_app.config['AUTH'].get( 'default', [['BROKEN'],['']] ) #Default rule when no specific rule
        )
        logging.warning( "Url requires access: {}".format(auth_rule) )

        auth.check_auth( *auth_rule )

        return f(*args, **kwargs)
    return decorated
<commit_msg>Improved logging.<commit_after>from flask import abort, request, current_app
from functools import wraps
import authorise as auth
import logging

def authenticate(f):
    """
    Decorator to require api authentication

    Args:
        f: flask function
    Returns:
       function: decorated function or abort(401)
    """
    @wraps(f)
    def decorated(*args, **kwargs):

        #Load the authentication rule from configs, based on the request url_rule.
        auth_rule = current_app.config['AUTH'].get(
            str(request.url_rule),
            current_app.config['AUTH'].get( 'default', [['BROKEN'],['']] ) #Default rule when no specific rule
        )
        logging.warning( "Url requires access: {}".format(auth_rule) )

        auth.check_auth( *auth_rule )

        return f(*args, **kwargs)
    return decorated
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import requests
import bs4

class Club:
	"""A MLS Club

	Required Attributes:
		name: The name of the club.
		abbreviation: The common abbreviation for the club.
		subreddit: The subreddit (per /r/mls) for the club.
	Optional Attributes:
		points: Total accumulated points.
		games_played: Number of games played.
		goal_difference: The positive or negative goal difference. This is
			the sum of goals for and goals against.
		goals_for: Total number of goals scored.
	"""

	def __init__(self, name, abbreviation, conference, subreddit, rank=0,
				points=0, games_played=0, goal_difference=0, goals_for=0):
		self.name = name
		self.abbreviation = abbreviation
		self.conference = conference
		self.subreddit = subreddit
		self.rank = rank
		self.points = points
		self.games_played = games_played
		self.goal_difference = goal_difference
		self.goals_for = goals_for

	def formatted(self):
		return '[%s](%s)' % (self.abbreviation, self.subreddit)

CHI  = Club('Chicago Fire',           'CHI',  'Eastern', '/r/chicagofire')
COL  = Club('Colorado Rapids',        'COL',  'Western', '/r/rapids')
CLB  = Club('Columbus Crew SC',       'CLB',  'Eastern', '/r/thremassive')
DC   = Club('D.C. United',            'DC',   'Eastern', '/r/dcunited')
FCD  = Club('FC Dallas',              'DAL',  'Western', '/r/fcdallas')
HOU  = Club('Houston Dynamo',         'HOU',  'Western', '/r/dynamo')
LAG  = Club('LA Galaxy',              'LA',   'Western', '/r/lagalaxy')
MTL  = Club('Montreal Impact',        'MTL',  'Eastern', '/r/montrealimpact')
NE   = Club('New England Revolution', 'NE',   'Eastern', '/r/newenglandrevolution')
NYC  = Club('New York City FC',       'NYC',  'Eastern', '/r/nycfc')
NYRB = Club('New York Red Bulls',     'NYRB', 'Eastern', '/r/rbny')
OCSC = Club('Orlando City SC',        'OCSC', 'Eastern', '/r/oclions')
PHI  = Club('Philadelphia Union',     'PHI',  'Eastern', '/r/phillyunion')
POR  = Club('Portland Timbers',       'POR',  'Western', '/r/timbers')
RSL  = Club('Real Salt Lake',         'RSL',  'Western', '/r/realsaltlake')
SJ   = Club('San Jose Earthquakes',   'SJ',   'Western', '/r/sjearthquakes')
SEA  = Club('Seattle Sounders FC',    'SEA',  'Western', '/r/soundersfc')
SKC  = Club('Sporting Kansas City',   'SKC',  'Western', '/r/sportingkc')
TOR  = Club('Toronto FC',             'TFC',  'Eastern', '/r/tfc')
VAN  = Club('Vancouver Whitecaps FC', 'VAN',  'Western', '/r/whitecapsfc')

ALL_CLUBS = {
	CHI, COL, CLB, DC, FCD, HOU, LAG, MTL, NE, NYC, NYRB, OCSC, PHI, POR, RSL,
	SJ, SEA, SKC, TOR, VAN
}

def strip_text(text):
	return text.get_text().strip();

def find_club(name):
	for club in ALL_CLUBS:
		if club.name.lower() == name.lower():
			return club

	sys.exit("Could not find club %s" % name)

def format_club(name):
	club = find_club(strip_text(name))
	return club.formatted()

def print_table(table):
	for row in table:
		cells = row.find_all('td')
		rank = strip_text(cells[0])
		club = format_club(cells[1].find('a'))
		points = '**%s**' % strip_text(cells[2])
		gp = strip_text(cells[3])
		gd = strip_text(cells[10])
		gf = strip_text(cells[8])
		print '%s|%s|%s|%s|%s|%s' % (rank, club, points, gp, gd, gf)

def main():
	# Setup our variables for URLs and data to parse
	standings_url = requests.get('http://www.mlssoccer.com/standings')
	standing_soup = bs4.BeautifulSoup(standings_url.text)

	# East Conference is the 1st table
	eastern_table = standing_soup.select('.stats-table')[0]
	eastern_data = eastern_table.find('tbody').find_all('tr')

	# Western Conference is the 2nd table
	western_table = standing_soup.select('.stats-table')[1]
	western_data = western_table.find('tbody').find_all('tr')

	print "=== Eastern Conference ==="
	print_table(eastern_data)
	print "\n=== Western Conference ==="
	print_table(western_data)

if __name__ == '__main__':
	main()
<commit_msg>setup: New method to setup club data<commit_after>#!/usr/bin/python
import sys
import requests
import bs4

class Club:
	"""A MLS Club

	Required Attributes:
		name: The name of the club.
		abbreviation: The common abbreviation for the club.
		subreddit: The subreddit (per /r/mls) for the club.
	Optional Attributes:
		points: Total accumulated points.
		games_played: Number of games played.
		goal_difference: The positive or negative goal difference. This is
			the sum of goals for and goals against.
		goals_for: Total number of goals scored.
	"""

	def __init__(self, name, abbreviation, conference, subreddit, rank=0,
				points=0, games_played=0, goal_difference=0, goals_for=0):
		self.name = name
		self.abbreviation = abbreviation
		self.conference = conference
		self.subreddit = subreddit
		self.rank = rank
		self.points = points
		self.games_played = games_played
		self.goal_difference = goal_difference
		self.goals_for = goals_for

	def formatted(self):
		return '[%s](%s)' % (self.abbreviation, self.subreddit)

CHI  = Club('Chicago Fire',           'CHI',  'Eastern', '/r/chicagofire')
COL  = Club('Colorado Rapids',        'COL',  'Western', '/r/rapids')
CLB  = Club('Columbus Crew SC',       'CLB',  'Eastern', '/r/thremassive')
DC   = Club('D.C. United',            'DC',   'Eastern', '/r/dcunited')
FCD  = Club('FC Dallas',              'DAL',  'Western', '/r/fcdallas')
HOU  = Club('Houston Dynamo',         'HOU',  'Western', '/r/dynamo')
LAG  = Club('LA Galaxy',              'LA',   'Western', '/r/lagalaxy')
MTL  = Club('Montreal Impact',        'MTL',  'Eastern', '/r/montrealimpact')
NE   = Club('New England Revolution', 'NE',   'Eastern', '/r/newenglandrevolution')
NYC  = Club('New York City FC',       'NYC',  'Eastern', '/r/nycfc')
NYRB = Club('New York Red Bulls',     'NYRB', 'Eastern', '/r/rbny')
OCSC = Club('Orlando City SC',        'OCSC', 'Eastern', '/r/oclions')
PHI  = Club('Philadelphia Union',     'PHI',  'Eastern', '/r/phillyunion')
POR  = Club('Portland Timbers',       'POR',  'Western', '/r/timbers')
RSL  = Club('Real Salt Lake',         'RSL',  'Western', '/r/realsaltlake')
SJ   = Club('San Jose Earthquakes',   'SJ',   'Western', '/r/sjearthquakes')
SEA  = Club('Seattle Sounders FC',    'SEA',  'Western', '/r/soundersfc')
SKC  = Club('Sporting Kansas City',   'SKC',  'Western', '/r/sportingkc')
TOR  = Club('Toronto FC',             'TFC',  'Eastern', '/r/tfc')
VAN  = Club('Vancouver Whitecaps FC', 'VAN',  'Western', '/r/whitecapsfc')

ALL_CLUBS = {
	CHI, COL, CLB, DC, FCD, HOU, LAG, MTL, NE, NYC, NYRB, OCSC, PHI, POR, RSL,
	SJ, SEA, SKC, TOR, VAN
}

def strip_text(text):
	return text.get_text().strip();

def find_club(name):
	for club in ALL_CLUBS:
		if club.name.lower() == name.lower():
			return club

	sys.exit("Could not find club %s" % name)

def format_club(name):
	club = find_club(strip_text(name))
	return club.formatted()

def print_table(table):
	for row in table:
		cells = row.find_all('td')
		rank = strip_text(cells[0])
		club = format_club(cells[1].find('a'))
		points = '**%s**' % strip_text(cells[2])
		gp = strip_text(cells[3])
		gd = strip_text(cells[10])
		gf = strip_text(cells[8])
		print '%s|%s|%s|%s|%s|%s' % (rank, club, points, gp, gd, gf)

def setup(data):
	for row in data:
		cells = row.find_all('td')
		rank = strip_text(cells[0])
		name = strip_text(cells[1].find('a'))
		points = strip_text(cells[2])
		gp = strip_text(cells[3])
		gd = strip_text(cells[10])
		gf = strip_text(cells[8])
		# Set Club's option attributes
		club = find_club(name)
		club.rank = rank
		club.points = points
		club.games_played = gp
		club.goal_difference = gd
		club.goals_for = gf

def main():
	# Setup our variables for URLs and data to parse
	standings_url = requests.get('http://www.mlssoccer.com/standings')
	standing_soup = bs4.BeautifulSoup(standings_url.text)

	# East Conference is the 1st table
	eastern_table = standing_soup.select('.stats-table')[0]
	eastern_data = eastern_table.find('tbody').find_all('tr')

	# Western Conference is the 2nd table
	western_table = standing_soup.select('.stats-table')[1]
	western_data = western_table.find('tbody').find_all('tr')

	print "=== Eastern Conference ==="
	print_table(eastern_data)
	print "\n=== Western Conference ==="
	print_table(western_data)

if __name__ == '__main__':
	main()
<|endoftext|>