<commit_before>#!/usr/bin/python3

import sys
import traceback
import re

import requests

from flask import (Flask, send_file, send_from_directory, redirect, request,
                   make_response)
from flask_restful import Resource, Api
from flask_restful.reqparse import RequestParser

from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

from werkzeug.middleware.proxy_fix import ProxyFix

from reserved import RESERVED_USERNAMES

app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
app.config['RESTFUL_JSON'] = {"indent": 4}

api = Api(app)

http_port = 8080

def validate_pubkey(value):
    if len(value) > 8192 or len(value) < 80:
        raise ValueError("Expected length between 80 and 8192 characters")

    value = value.replace("\"", "").replace("'", "").replace("\\\"", "")
    value = value.split(' ')
    types = ['ecdsa-sha2-nistp256', 'ecdsa-sha2-nistp384',
             'ecdsa-sha2-nistp521', 'ssh-rsa', 'ssh-dss', 'ssh-ed25519']
    if value[0] not in types:
        raise ValueError(
            "Expected " + ', '.join(types[:-1]) + ', or ' + types[-1]
        )

    return "%s %s" % (value[0], value[1])

def validate_username(value):

    # Regexp must be kept in sync with
    #  https://github.com/hashbang/hashbang.sh/blob/master/src/hashbang.sh#L186-196
    if re.compile(r"^[a-z][a-z0-9]{,30}$").match(value) is None:
        raise ValueError('Username is invalid')

    if value in RESERVED_USERNAMES:
        raise ValueError('Username is reserved')

    return value

@api.representation('text/plain')
def output_plain(data, code, headers=None):
    lines = []
    for server in data:
        line = [
            server['name'],
            '',
            'DE',
            'N/A',
            str(server['maxusers'])
        ]
    lines.append("|".join(line))
    data = "\n".join(lines)
    resp = make_response(data, code)
    resp.headers.extend(headers or {})
    return resp

class UserCreate(Resource):
    def __init__(self):
        self.reqparse = RequestParser()
        self.reqparse.add_argument(
            'user',
            type=validate_username,
            required=True,
            location='json'
        )
        self.reqparse.add_argument(
            'key',
            type=validate_pubkey,
            required=True,
            location='json'
        )
        self.reqparse.add_argument(
            'host',
            type=str,
            required=True,
            location='json'
        )
        super(UserCreate, self).__init__()

    def post(self):
        args = self.reqparse.parse_args()

        try:
            post_data = {
                "name": str(args["user"]),
                "host": args["host"],
                "data": {
                    "shell": "/bin/bash",
                    "ssh_keys": [args["key"]]
                }
            }
            r = requests.post(
                "http://userdb-api.userdb-api.svc/passwd",
                json=post_data
            )
            print("{status_code} - {user} - {text}".format(
                status_code=r.status_code,
                user=args['user'],
                text=r.text
            ))
            if r.status_code >= 400:
                data = r.json()
                return {'message': data['message']}, r.status_code
        except Exception as e:
            (typ, value, tb) = sys.exc_info()
            sys.stderr.write("Unexpected Error: %r\n" % e)
            traceback.print_tb(tb)
            return {'message': 'User creation script failed'}, 500

        return {'message': 'success'}

class ServerStats(Resource):
    LOCATIONS = {
        "da1.hashbang.sh": {"lat": 32.8, "lon": -96.8},
        "ny1.hashbang.sh": {"lat": 40.7, "lon": -74},
        "sf1.hashbang.sh": {"lat": 37.8, "lon": -122.4},
        "to1.hashbang.sh": {"lat": 43.7, "lon": -79.4},
        "de1.hashbang.sh": {"lat": 49.4478, "lon": 11.0683},
    }

    def get(self, out_format='json'):
        try:
            data = requests.get("http://userdb-api.userdb-api.svc/hosts").json()
        except Exception as e:  # noqa
            return {'message': 'Unable to connect to server'}, 500

        for idx, s in enumerate(data):
            server_host = s['name']
            if server_host in self.LOCATIONS.keys():
                data[idx].update({'coordinates': self.LOCATIONS[server_host]})

        return data

api.add_resource(UserCreate, '/user/create')
api.add_resource(ServerStats, '/server/stats')

def security_headers(response, secure=False):
    csp = "default-src 'none'; " \
          "style-src https://fonts.googleapis.com 'self'; " \
          "font-src https://fonts.gstatic.com; " \
          "img-src data:; script-src 'self'; " \
          "sandbox allow-same-origin allow-scripts; " \
          "frame-ancestors 'none'"

    response.headers['Content-Security-Policy'] = csp
    response.headers['Referrer-Policy'] = 'no-referrer'
    response.headers['X-Content-Type-Options'] = 'nosniff'
    response.headers['X-Frame-Options'] = 'DENY'
    response.headers['X-XSS-Protection'] = '1; mode=block'

    if secure:
        response.headers['Strict-Transport-Security'] = 'max-age=31536000'

    return response

@app.route('/', methods=["GET"])
def root():
    useragent = request.headers.get('User-Agent')
    has_https = 'https_server' in globals()

    if 'curl' in useragent and not request.is_secure:
        resp = send_from_directory('static', 'warn.sh.asc')
    elif not has_https or request.is_secure:
        resp = send_from_directory('static', 'index.html')
    else:
        return redirect(request.url.replace("http://", "https://"))

    return security_headers(resp, secure=request.is_secure)

@app.route('/LICENSE.md', methods=['GET'])
def license():
    return security_headers(send_file('LICENSE.md', mimetype='text/markdown'),
                            secure=request.is_secure)

# HE.net domain validation
@app.route('/s73rmwh.txt', methods=['GET'])
def he_net():
    return security_headers(make_response('Hello IPv6!'),
                            secure=request.is_secure)

@app.route('/assets/<path:filename>', methods=['GET'])
def assets(filename):
    return security_headers(send_from_directory('src', filename),
                            secure=request.is_secure)

if __name__ == '__main__':
    http_server = HTTPServer(WSGIContainer(app))
    http_server.listen(http_port)
    IOLoop.instance().start()
<commit_msg>server: Send HTTPS redirect if User-Agent header is missing<commit_after>#!/usr/bin/python3

import sys
import traceback
import re

import requests

from flask import (Flask, send_file, send_from_directory, redirect, request,
                   make_response)
from flask_restful import Resource, Api
from flask_restful.reqparse import RequestParser

from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

from werkzeug.middleware.proxy_fix import ProxyFix

from reserved import RESERVED_USERNAMES

app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
app.config['RESTFUL_JSON'] = {"indent": 4}

api = Api(app)

http_port = 8080

def validate_pubkey(value):
    if len(value) > 8192 or len(value) < 80:
        raise ValueError("Expected length between 80 and 8192 characters")

    value = value.replace("\"", "").replace("'", "").replace("\\\"", "")
    value = value.split(' ')
    types = ['ecdsa-sha2-nistp256', 'ecdsa-sha2-nistp384',
             'ecdsa-sha2-nistp521', 'ssh-rsa', 'ssh-dss', 'ssh-ed25519']
    if value[0] not in types:
        raise ValueError(
            "Expected " + ', '.join(types[:-1]) + ', or ' + types[-1]
        )

    return "%s %s" % (value[0], value[1])

def validate_username(value):

    # Regexp must be kept in sync with
    #  https://github.com/hashbang/hashbang.sh/blob/master/src/hashbang.sh#L186-196
    if re.compile(r"^[a-z][a-z0-9]{,30}$").match(value) is None:
        raise ValueError('Username is invalid')

    if value in RESERVED_USERNAMES:
        raise ValueError('Username is reserved')

    return value

@api.representation('text/plain')
def output_plain(data, code, headers=None):
    lines = []
    for server in data:
        line = [
            server['name'],
            '',
            'DE',
            'N/A',
            str(server['maxusers'])
        ]
    lines.append("|".join(line))
    data = "\n".join(lines)
    resp = make_response(data, code)
    resp.headers.extend(headers or {})
    return resp

class UserCreate(Resource):
    def __init__(self):
        self.reqparse = RequestParser()
        self.reqparse.add_argument(
            'user',
            type=validate_username,
            required=True,
            location='json'
        )
        self.reqparse.add_argument(
            'key',
            type=validate_pubkey,
            required=True,
            location='json'
        )
        self.reqparse.add_argument(
            'host',
            type=str,
            required=True,
            location='json'
        )
        super(UserCreate, self).__init__()

    def post(self):
        args = self.reqparse.parse_args()

        try:
            post_data = {
                "name": str(args["user"]),
                "host": args["host"],
                "data": {
                    "shell": "/bin/bash",
                    "ssh_keys": [args["key"]]
                }
            }
            r = requests.post(
                "http://userdb-api.userdb-api.svc/passwd",
                json=post_data
            )
            print("{status_code} - {user} - {text}".format(
                status_code=r.status_code,
                user=args['user'],
                text=r.text
            ))
            if r.status_code >= 400:
                data = r.json()
                return {'message': data['message']}, r.status_code
        except Exception as e:
            (typ, value, tb) = sys.exc_info()
            sys.stderr.write("Unexpected Error: %r\n" % e)
            traceback.print_tb(tb)
            return {'message': 'User creation script failed'}, 500

        return {'message': 'success'}

class ServerStats(Resource):
    LOCATIONS = {
        "da1.hashbang.sh": {"lat": 32.8, "lon": -96.8},
        "ny1.hashbang.sh": {"lat": 40.7, "lon": -74},
        "sf1.hashbang.sh": {"lat": 37.8, "lon": -122.4},
        "to1.hashbang.sh": {"lat": 43.7, "lon": -79.4},
        "de1.hashbang.sh": {"lat": 49.4478, "lon": 11.0683},
    }

    def get(self, out_format='json'):
        try:
            data = requests.get("http://userdb-api.userdb-api.svc/hosts").json()
        except Exception as e:  # noqa
            return {'message': 'Unable to connect to server'}, 500

        for idx, s in enumerate(data):
            server_host = s['name']
            if server_host in self.LOCATIONS.keys():
                data[idx].update({'coordinates': self.LOCATIONS[server_host]})

        return data

api.add_resource(UserCreate, '/user/create')
api.add_resource(ServerStats, '/server/stats')

def security_headers(response, secure=False):
    csp = "default-src 'none'; " \
          "style-src https://fonts.googleapis.com 'self'; " \
          "font-src https://fonts.gstatic.com; " \
          "img-src data:; script-src 'self'; " \
          "sandbox allow-same-origin allow-scripts; " \
          "frame-ancestors 'none'"

    response.headers['Content-Security-Policy'] = csp
    response.headers['Referrer-Policy'] = 'no-referrer'
    response.headers['X-Content-Type-Options'] = 'nosniff'
    response.headers['X-Frame-Options'] = 'DENY'
    response.headers['X-XSS-Protection'] = '1; mode=block'

    if secure:
        response.headers['Strict-Transport-Security'] = 'max-age=31536000'

    return response

@app.route('/', methods=["GET"])
def root():
    useragent = request.headers.get('User-Agent', '')
    has_https = 'https_server' in globals()

    if 'curl' in useragent and not request.is_secure:
        resp = send_from_directory('static', 'warn.sh.asc')
    elif not has_https or request.is_secure:
        resp = send_from_directory('static', 'index.html')
    else:
        return redirect(request.url.replace("http://", "https://"))

    return security_headers(resp, secure=request.is_secure)

@app.route('/LICENSE.md', methods=['GET'])
def license():
    return security_headers(send_file('LICENSE.md', mimetype='text/markdown'),
                            secure=request.is_secure)

# HE.net domain validation
@app.route('/s73rmwh.txt', methods=['GET'])
def he_net():
    return security_headers(make_response('Hello IPv6!'),
                            secure=request.is_secure)

@app.route('/assets/<path:filename>', methods=['GET'])
def assets(filename):
    return security_headers(send_from_directory('src', filename),
                            secure=request.is_secure)

if __name__ == '__main__':
    http_server = HTTPServer(WSGIContainer(app))
    http_server.listen(http_port)
    IOLoop.instance().start()
<|endoftext|>


<commit_before>#!/usr/bin/python3

'''
=== PREREQUISITES ===
Run in Python 3 or Python 2

Install requests library, via macOS terminal:
sudo pip install requests

login.py has these two lines, with the API key from your Dashboard profile (upper-right email login > API access), and organization ID to call (https://dashboard.meraki.com/api/v0/organizations); separated into different file for security.
api_key = '[API_KEY]'
org_id = '[ORG_ID]'

Usage:
python uplink.py

=== DESCRIPTION ===
Iterates through all devices, and exports to two CSV files: one for appliance (MX, Z1, vMX100) networks to collect WAN uplink information, and the other for all other devices (MR, MS, MC, MV) with local uplink info.

Possible statuses:
Active: active and working WAN port
Ready: standby but working WAN port, not the preferred WAN port
Failed: was working at some point but not anymore
Not connected: nothing was ever connected, no cable plugged in
For load balancing, both WAN links would show active.

For any questions, please contact Shiyue (Shay) Cheng, shiychen@cisco.com
'''

import csv
import datetime
import json
import requests

def get_network_name(network_id, networks):
    return [element for element in networks if network_id == element['id']][0]['name']

if __name__ == '__main__':
    # Import API key and org ID from login.py
    try:
        import login
        (API_KEY, ORG_ID) = (login.api_key, login.org_id)
    except ImportError:
        try: input = raw_input
        except NameError: pass
        API_KEY = input('Enter your Dashboard API key: ')
        ORG_ID = input('Enter your organization ID: ')

    # Find all appliance networks (MX, Z1, vMX100)
    session = requests.session()
    headers = {'X-Cisco-Meraki-API-Key': API_KEY, 'Content-Type': 'application/json'}
    name = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID, headers=headers).text)['name']
    networks = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID + '/networks', headers=headers).text)
    inventory = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID + '/inventory', headers=headers).text)
    appliances = [device for device in inventory if device['model'][:2] in ('MX', 'Z1', 'vM') and device['networkId'] is not None]
    devices = [device for device in inventory if device not in appliances and device['networkId'] is not None]

    # Output CSV of appliances' info
    today = datetime.date.today()
    csv_file1 = open(name + ' MX & Z1 appliances -' + str(today) + '.csv', 'w')
    fieldnames = ['Network', 'Device', 'Serial', 'MAC', 'Model', 'WAN1 Status', 'WAN1 IP', 'WAN1 Gateway', 'WAN1 Public IP', 'WAN1 DNS', 'WAN1 Static', 'WAN2 Status', 'WAN2 IP', 'WAN2 Gateway', 'WAN2 Public IP', 'WAN2 DNS', 'WAN2 Static', 'Cellular Status', 'Cellular IP', 'Cellular Provider', 'Cellular Public IP', 'Cellular Model', 'Cellular Connection']
    writer = csv.DictWriter(csv_file1, fieldnames=fieldnames, restval='')
    writer.writeheader()

    # Iterate through appliances
    for appliance in appliances:
        network_name = get_network_name(appliance['networkId'], networks)
        device_name = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + appliance['networkId'] + '/devices/' + appliance['serial'], headers=headers).text)['name']
        try:
            print('Found appliance ' + device_name)
        except:
            print('Found appliance ' + appliance['serial'])
        uplinks_info = dict.fromkeys(['WAN1', 'WAN2', 'Cellular'])
        uplinks_info['WAN1'] = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'usingStaticIp'])
        uplinks_info['WAN2'] = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'usingStaticIp'])
        uplinks_info['Cellular'] = dict.fromkeys(['interface', 'status', 'ip', 'provider', 'publicIp', 'model', 'connectionType'])
        uplinks = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + appliance['networkId'] + '/devices/' + appliance['serial'] + '/uplink', headers=headers).text)
        for uplink in uplinks:
            if uplink['interface'] == 'WAN 1':
                for key in uplink.keys():
                    uplinks_info['WAN1'][key] = uplink[key]
            elif uplink['interface'] == 'WAN 2':
                for key in uplink.keys():
                    uplinks_info['WAN2'][key] = uplink[key]
            elif uplink['interface'] == 'Cellular':
                for key in uplink.keys():
                    uplinks_info['Cellular'][key] = uplink[key]
        writer.writerow({'Network': network_name, 'Device': device_name, 'Serial': appliance['serial'], 'MAC': appliance['mac'], 'Model': appliance['model'], 'WAN1 Status': uplinks_info['WAN1']['status'], 'WAN1 IP': uplinks_info['WAN1']['ip'], 'WAN1 Gateway': uplinks_info['WAN1']['gateway'], 'WAN1 Public IP': uplinks_info['WAN1']['publicIp'], 'WAN1 DNS': uplinks_info['WAN1']['dns'], 'WAN1 Static': uplinks_info['WAN1']['usingStaticIp'], 'WAN2 Status': uplinks_info['WAN2']['status'], 'WAN2 IP': uplinks_info['WAN2']['ip'], 'WAN2 Gateway': uplinks_info['WAN2']['gateway'], 'WAN2 Public IP': uplinks_info['WAN2']['publicIp'], 'WAN2 DNS': uplinks_info['WAN2']['dns'], 'WAN2 Static': uplinks_info['WAN2']['usingStaticIp'], 'Cellular Status': uplinks_info['Cellular']['status'], 'Cellular IP': uplinks_info['Cellular']['ip'], 'Cellular Provider': uplinks_info['Cellular']['provider'], 'Cellular Public IP': uplinks_info['Cellular']['publicIp'], 'Cellular Model': uplinks_info['Cellular']['model'], 'Cellular Connection': uplinks_info['Cellular']['connectionType']})
    csv_file1.close()

    # Output CSV of all other devices' info
    csv_file2 = open(name + ' other devices -' + str(today) + '.csv', 'w')
    fieldnames = ['Network', 'Device', 'Serial', 'MAC', 'Model', 'Status', 'IP', 'Gateway', 'Public IP', 'DNS', 'VLAN', 'Static']
    writer = csv.DictWriter(csv_file2, fieldnames=fieldnames, restval='')
    writer.writeheader()

    # Iterate through all other devices
    for device in devices:
        network_name = get_network_name(device['networkId'], networks)
        device_name = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + device['networkId'] + '/devices/' + device['serial'], headers=headers).text)['name']
        try:
            print('Found device ' + device_name)
        except:
            print('Found device ' + device['serial'])
        uplink_info = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'vlan', 'usingStaticIp'])
        uplink = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + device['networkId'] + '/devices/' + device['serial'] + '/uplink', headers=headers).text)

        # Blank uplink for devices that are down or meshed APs
        if uplink == []:
            continue
        # All other devices have single uplink
        else:
            uplink = uplink[0]
        for key in uplink.keys():
            uplink_info[key] = uplink[key]
        writer.writerow({'Network': network_name, 'Device': device_name, 'Serial': device['serial'], 'MAC': device['mac'], 'Model': device['model'], 'Status': uplink_info['status'], 'IP': uplink_info['ip'], 'Gateway': uplink_info['gateway'], 'Public IP': uplink_info['publicIp'], 'DNS': uplink_info['dns'], 'VLAN': uplink_info['vlan'], 'Static': uplink_info['usingStaticIp']})
    csv_file2.close()
<commit_msg>Fixed bug with non-ASCII characters<commit_after>#!/usr/bin/python3

'''
=== PREREQUISITES ===
Run in Python 3 or Python 2

Install requests library, via macOS terminal, depending on version of Python:
sudo pip3 install requests
sudo pip install requests

login.py has these two lines, with the API key from your Dashboard profile (upper-right email login > API access), and organization ID to call (https://dashboard.meraki.com/api/v0/organizations); separated into different file for security.
api_key = '[API_KEY]'
org_id = '[ORG_ID]'

Usage, depending on version of Python:
python3 uplink.py
python uplink.py

=== DESCRIPTION ===
Iterates through all devices, and exports to two CSV files: one for appliance (MX, Z1, vMX100) networks to collect WAN uplink information, and the other for all other devices (MR, MS, MC, MV) with local uplink info.

Possible statuses:
Active: active and working WAN port
Ready: standby but working WAN port, not the preferred WAN port
Failed: was working at some point but not anymore
Not connected: nothing was ever connected, no cable plugged in
For load balancing, both WAN links would show active.

For any questions, please contact Shiyue (Shay) Cheng, shiychen@cisco.com
'''

import csv
import datetime
import json
import requests

def get_network_name(network_id, networks):
    return [element for element in networks if network_id == element['id']][0]['name']

if __name__ == '__main__':
    # Import API key and org ID from login.py
    try:
        import login
        (API_KEY, ORG_ID) = (login.api_key, login.org_id)
    except ImportError:
        try: input = raw_input
        except NameError: pass
        API_KEY = input('Enter your Dashboard API key: ')
        ORG_ID = input('Enter your organization ID: ')

    # Find all appliance networks (MX, Z1, vMX100)
    session = requests.session()
    headers = {'X-Cisco-Meraki-API-Key': API_KEY, 'Content-Type': 'application/json'}
    name = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID, headers=headers).text)['name']
    networks = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID + '/networks', headers=headers).text)
    inventory = json.loads(session.get('https://dashboard.meraki.com/api/v0/organizations/' + ORG_ID + '/inventory', headers=headers).text)
    appliances = [device for device in inventory if device['model'][:2] in ('MX', 'Z1', 'vM') and device['networkId'] is not None]
    devices = [device for device in inventory if device not in appliances and device['networkId'] is not None]

    # Output CSV of appliances' info
    today = datetime.date.today()
    csv_file1 = open(name + ' MX & Z1 appliances -' + str(today) + '.csv', 'w', encoding='utf-8')
    fieldnames = ['Network', 'Device', 'Serial', 'MAC', 'Model', 'WAN1 Status', 'WAN1 IP', 'WAN1 Gateway', 'WAN1 Public IP', 'WAN1 DNS', 'WAN1 Static', 'WAN2 Status', 'WAN2 IP', 'WAN2 Gateway', 'WAN2 Public IP', 'WAN2 DNS', 'WAN2 Static', 'Cellular Status', 'Cellular IP', 'Cellular Provider', 'Cellular Public IP', 'Cellular Model', 'Cellular Connection']
    writer = csv.DictWriter(csv_file1, fieldnames=fieldnames, restval='')
    writer.writeheader()

    # Iterate through appliances
    for appliance in appliances:
        network_name = get_network_name(appliance['networkId'], networks)
        device_name = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + appliance['networkId'] + '/devices/' + appliance['serial'], headers=headers).text)['name']
        try:
            print('Found appliance ' + device_name)
        except:
            print('Found appliance ' + appliance['serial'])
        uplinks_info = dict.fromkeys(['WAN1', 'WAN2', 'Cellular'])
        uplinks_info['WAN1'] = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'usingStaticIp'])
        uplinks_info['WAN2'] = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'usingStaticIp'])
        uplinks_info['Cellular'] = dict.fromkeys(['interface', 'status', 'ip', 'provider', 'publicIp', 'model', 'connectionType'])
        uplinks = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + appliance['networkId'] + '/devices/' + appliance['serial'] + '/uplink', headers=headers).text)
        for uplink in uplinks:
            if uplink['interface'] == 'WAN 1':
                for key in uplink.keys():
                    uplinks_info['WAN1'][key] = uplink[key]
            elif uplink['interface'] == 'WAN 2':
                for key in uplink.keys():
                    uplinks_info['WAN2'][key] = uplink[key]
            elif uplink['interface'] == 'Cellular':
                for key in uplink.keys():
                    uplinks_info['Cellular'][key] = uplink[key]
        writer.writerow({'Network': network_name, 'Device': device_name, 'Serial': appliance['serial'], 'MAC': appliance['mac'], 'Model': appliance['model'], 'WAN1 Status': uplinks_info['WAN1']['status'], 'WAN1 IP': uplinks_info['WAN1']['ip'], 'WAN1 Gateway': uplinks_info['WAN1']['gateway'], 'WAN1 Public IP': uplinks_info['WAN1']['publicIp'], 'WAN1 DNS': uplinks_info['WAN1']['dns'], 'WAN1 Static': uplinks_info['WAN1']['usingStaticIp'], 'WAN2 Status': uplinks_info['WAN2']['status'], 'WAN2 IP': uplinks_info['WAN2']['ip'], 'WAN2 Gateway': uplinks_info['WAN2']['gateway'], 'WAN2 Public IP': uplinks_info['WAN2']['publicIp'], 'WAN2 DNS': uplinks_info['WAN2']['dns'], 'WAN2 Static': uplinks_info['WAN2']['usingStaticIp'], 'Cellular Status': uplinks_info['Cellular']['status'], 'Cellular IP': uplinks_info['Cellular']['ip'], 'Cellular Provider': uplinks_info['Cellular']['provider'], 'Cellular Public IP': uplinks_info['Cellular']['publicIp'], 'Cellular Model': uplinks_info['Cellular']['model'], 'Cellular Connection': uplinks_info['Cellular']['connectionType']})
    csv_file1.close()

    # Output CSV of all other devices' info
    csv_file2 = open(name + ' other devices -' + str(today) + '.csv', 'w', encoding='utf-8')
    fieldnames = ['Network', 'Device', 'Serial', 'MAC', 'Model', 'Status', 'IP', 'Gateway', 'Public IP', 'DNS', 'VLAN', 'Static']
    writer = csv.DictWriter(csv_file2, fieldnames=fieldnames, restval='')
    writer.writeheader()

    # Iterate through all other devices
    for device in devices:
        network_name = get_network_name(device['networkId'], networks)
        device_name = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + device['networkId'] + '/devices/' + device['serial'], headers=headers).text)['name']
        try:
            print('Found device ' + device_name)
        except:
            print('Found device ' + device['serial'])
        uplink_info = dict.fromkeys(['interface', 'status', 'ip', 'gateway', 'publicIp', 'dns', 'vlan', 'usingStaticIp'])
        uplink = json.loads(session.get('https://dashboard.meraki.com/api/v0/networks/' + device['networkId'] + '/devices/' + device['serial'] + '/uplink', headers=headers).text)

        # Blank uplink for devices that are down or meshed APs
        if uplink == []:
            continue
        # All other devices have single uplink
        else:
            uplink = uplink[0]
        for key in uplink.keys():
            uplink_info[key] = uplink[key]
        writer.writerow({'Network': network_name, 'Device': device_name, 'Serial': device['serial'], 'MAC': device['mac'], 'Model': device['model'], 'Status': uplink_info['status'], 'IP': uplink_info['ip'], 'Gateway': uplink_info['gateway'], 'Public IP': uplink_info['publicIp'], 'DNS': uplink_info['dns'], 'VLAN': uplink_info['vlan'], 'Static': uplink_info['usingStaticIp']})
    csv_file2.close()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function

import argparse
import sys
import textwrap
import os.path
import re
import pandas as pd

from datetime import date, datetime, timedelta

try:
  from lxml import html
  import requests
except ImportError as exception:
  print('[-] This script needs the lxml and requests library: {0}'.format(exception), file=sys.stderr)
  if sys.platform == 'win32' or sys.platform == 'cygwin':
    print("With windows, you will need to download lxml, see README.md documentation", file=sys.stderr)
  else:
    print("Install lxml with: sudo pip install lxml", file=sys.stderr)

  print("Install requests with: sudo pip install requests", file=sys.stderr)
  sys.exit(-1)

BASE_URL = 'https://www.us-cert.gov/ncas/bulletins/'

def get_bulletin_name(date_object=date.today()):
  '''
  return correct filename of a bulletin for a given data, published on Mondays
  '''
  while date_object.weekday():
    date_object = date_object - timedelta(days=1)
  return 'SB{0:%y}-{1:%j}'.format(date_object, date_object)

def get_bulletin_list(options):
  #
  # returns a list of all filenames of bulletins to retrieve
  #
  bulletin_list = []
  from_date = options['from_date']

  while from_date <= (options['to_date'] + timedelta(days=3)):
    bulletin_list.append(get_bulletin_name(from_date))
    from_date += timedelta(weeks=1)
  return bulletin_list

def retrieve_bulletin(filename, bulletin_name, options):
  '''
  returns a bulletin as a tree or an empty object if bulletin cannot be read
  returns bulletin via HTTP or directory file if we have it localized already
  '''
  tree = None
  url = '{0}{1}'.format(BASE_URL, bulletin_name)
  try:
    if options['force'] or not os.path.isfile(filename):
      page = requests.get(url)
      if page.status_code == 403:
        url += '-0'
        page = requests.get(url)
      if page.status_code == 200:
        tree = html.fromstring(page.text)
        if check_title(tree):
          with open(filename, 'wb') as html_page:
            html_page.write(page.text.encode('utf-8'))
    else:
      with open(filename, 'r') as html_page:
        try:
          tree = html.fromstring(html_page.read())
        except:
          print ('\nError with ' + filename)
    date_object = check_title(tree)
    if date_object and (date_object > options['from_date']) or options['latest']:
      return tree
    else:
      return None
  except requests.exceptions.ConnectionError as exception:
    sys.exit(exception.errno)

def setup_options(options):
  '''
  setup options for the rest of the program
  '''
  if not options['directory'].endswith(os.path.sep):
    options['directory'] += os.path.sep
  if not os.path.exists(options['directory']):
    os.makedirs(options['directory'])
  if not os.path.exists(options['directory']):
    os.makedirs(options['directory'])
  if not os.path.exists(options['tables']):
    os.makedirs(options['tables'])
  try:
    if options['from_date']:
      options['from_date'] = datetime.strptime(options['from_date'], '%d-%m-%Y').date()
  except ValueError:
    logging.error('Dates must be in the form dd-mm-YYYY')
    sys.exit(-1)
  options['to_date'] = date.today() + timedelta(days=2)
  options['selection'] = ['High']
  if not options['from_date'] and ('year' not in options or (options['update'])):
    options['from_date'] = date(date.today().year, 1, 1)
  else:
    if options['year'] >= 2010 and options['year'] <= date.today().year:
      options['from_date'] = date(options['year'], 1, 1)
      options['to_date'] = date(options['year'], 12, 31)
  if options['all']:
    options['from_date'] = date(2010, 1, 1)
  if options['from_date'] is None:
    options['from_date'] = options['to_date'] - timedelta(days=7)
  if options['low']:
    options['selection'].append('Low')
  if options['medium']:
    options['selection'].append('Medium')
  if options['unassigned']:
    options['selection'].append('Severity Not Yet Assigned')
  return options

def check_title(tree):
  '''
  check if tree contains vulnerability summary.
  returns a date object if successful.
  '''
  try:
    title = tree.xpath('//title/text()')[0].split(' | ')[0].replace('[\'', '')
  except AttributeError:
    title = ''

  response = re.search(r'Vulnerability Summary for the Week of (\w+\s[0-9]{1,2}\,\s[0-9]{4})', title)
  if response:
    date_object = datetime.strptime(response.group(1), '%B %d, %Y')
    return date_object.date()
  else:
    return False

def make_csv_files(tree, vuln_type, bulletin_name, options):
  '''
  takes selections and grabs html and converts to csv
  '''
  vulnerabilities = list()
  headers = ['Vendor', 'Product', 'Description', 'Published', 'CVSS', 'CVSS Score', 'Source Info']
  vendor = ''
  product = ''
  summary = 'Severity Not Yet Assigned' if vuln_type == 'Severity Not Yet Assigned' else '{0} Vulnerabilities'.format(vuln_type)

  # create vulnerability array
  for vuln in tree.xpath('//table[@summary="{0}"]/tbody/tr'.format(summary)):
    if len(vuln) > 1:
      try:
        try:
          vendor, product = vuln[0].text.split(' -- ')
        except AttributeError:
          # on occasion, US-CERT throws in random
 tags and it screws with the parsing
          vendor, product = vuln[0][0].text.split(' -- ')
      except ValueError:
        vendor = vuln[0].text
      description = vuln[1].text
      published = vuln[2].text
      cvss = vuln[4][0].text
      try:
        cvss_score = vuln[3][0].text
      except IndexError:
        cvss_score = '0'

      source_info = build_links(vuln[4], options['link'])
      #source_info = (b' '.join(list(map(lambda x:html.tostring(x), vuln[4])))).decode("utf-8")
      current_vuln = [vendor, product, description, published, cvss, cvss_score, source_info]
      vulnerabilities.append(current_vuln)

  df = pd.DataFrame(vulnerabilities, columns=headers)
  filename = '{0}/{1} - {2} Vulnerabilities.csv'.format(options['tables'], bulletin_name, vuln_type)
  df.to_csv(filename, index=False, encoding='utf-8')

def build_links(element, link_type):
  '''
  build links from source & patch info column
  '''
  source = ''

  if link_type == 'a':
    # loop if requesting anchor tags
    for i in range(0, len(element), 2):
      source_info_href = element[i].get("href")
      source_info_text = element[i].text
      source += '{1}
'.format(source_info_href, source_info_text)
  else:
    # cannot loop as csvs do not support multiple hyperlinks (particularly excel)
    source_info_href = element[0].get("href")
    source_info_text = element[0].text
    source += '=HYPERLINK("{0}","{1}")'.format(source_info_href, source_info_text)
  return source

def parse_arguments():
  '''
  parse command line args, exits on invalid args
  '''
  parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=textwrap.dedent('''\
    Downloads and parses vulnerability summaries from the US-CERT website.
    Creates CSV file(s) for further dissemination.

    Copyright (C) 2017 Adam Schaal
    MIT License'''))

  parser.add_argument('-a', '--all', action='store_true', help='retrieve all missing bulletin since 2010 (lengthy!)')
  parser.add_argument('-b', '--bulletin', action='store_true', help='retrieve a specific bulletin')
  parser.add_argument('-c', '--csv', action='store_true', help='creates csv files from html')
  parser.add_argument('-d', '--directory', action='store', default='bulletins', help='name of directory for saving bulletins - default(bulletins)')
  parser.add_argument('-f', '--force', action='store_true', help='force download, ignore / overwrite cached directory')
  parser.add_argument('--from-date', action='store', help='starting date (dd-mm-YYYY)')
  parser.add_argument('-l', '--low', action='store_true', help='select low vulnerabilities as well')
  parser.add_argument('--latest', action='store_true', help='show latest bulletin')
  parser.add_argument('--link', action='store', help='choose "a" for anchor tags or "h" for csv hyperlinks - default(h)')
  parser.add_argument('-m', '--medium', action='store_true', help='show medium vulnerabilities as well')
  parser.add_argument('-t', '--tables', action='store', default='tables', help='name of directory for saving tables - default(tables)')
  parser.add_argument('-u', '--unassigned', action='store_true', help='show Severity Not Yet Assigned vulnerabilities as well')
  parser.add_argument('--update', action='store_true', help='retrieve all newest bulletin since last update')
  parser.add_argument('--year', action='store', nargs='?', default=0, type=int, help='retrieve all bulletins for a given year year')

  return vars(parser.parse_args())

def main():
  '''
  main loop
  '''
  options = parse_arguments()
  setup_options(options)

  for bulletin_name in get_bulletin_list(options):
    if options['bulletin']:
      bulletin_name = options['bulletin']
    filename = '{0}{1}.html'.format(options['directory'], bulletin_name)
    tree = retrieve_bulletin(filename, bulletin_name, options)
    if tree is not None and options['csv']:
      for vuln_type in options['selection']:
        make_csv_files(tree, vuln_type, bulletin_name, options)

if __name__ == "__main__":
	main()<commit_msg>Small additions for py2 and 3<commit_after>from __future__ import absolute_import
from __future__ import print_function

import argparse
import sys
import textwrap
import os.path
import re
import pandas as pd

from datetime import date, datetime, timedelta
try:
    from urllib import quote_plus  # Python 2.X
except ImportError:
    from urllib.parse import quote_plus  # Python 3+

try:
  from lxml import html
  import requests
except ImportError as exception:
  print('[-] This script needs the lxml and requests library: {0}'.format(exception), file=sys.stderr)
  if sys.platform == 'win32' or sys.platform == 'cygwin':
    print("With windows, you will need to download lxml, see README.md documentation", file=sys.stderr)
  else:
    print("Install lxml with: sudo pip install lxml", file=sys.stderr)

  print("Install requests with: sudo pip install requests", file=sys.stderr)
  sys.exit(-1)

BASE_URL = 'https://www.us-cert.gov/ncas/bulletins/'

def get_bulletin_name(date_object=date.today()):
  '''
  return correct filename of a bulletin for a given data, published on Mondays
  '''
  while date_object.weekday():
    date_object = date_object - timedelta(days=1)
  return 'SB{0:%y}-{1:%j}'.format(date_object, date_object)

def get_bulletin_list(options):
  #
  # returns a list of all filenames of bulletins to retrieve
  #
  bulletin_list = []
  from_date = options['from_date']

  while from_date <= (options['to_date'] + timedelta(days=3)):
    bulletin_list.append(get_bulletin_name(from_date))
    from_date += timedelta(weeks=1)
  return bulletin_list

def retrieve_bulletin(filename, bulletin_name, options):
  '''
  returns a bulletin as a tree or an empty object if bulletin cannot be read
  returns bulletin via HTTP or directory file if we have it localized already
  '''
  tree = None
  url = '{0}{1}'.format(BASE_URL, bulletin_name)
  try:
    if options['force'] or not os.path.isfile(filename):
      page = requests.get(url)
      if page.status_code == 403:
        url += '-0'
        page = requests.get(url)
      if page.status_code == 200:
        tree = html.fromstring(page.text)
        if check_title(tree):
          with open(filename, 'wb') as html_page:
            html_page.write(page.text.encode('utf-8'))
    else:
      with open(filename, 'r') as html_page:
        try:
          tree = html.fromstring(html_page.read())
        except:
          print ('\nError with ' + filename)
    date_object = check_title(tree)
    if date_object and (date_object > options['from_date']) or options['latest']:
      return tree
    else:
      return None
  except requests.exceptions.ConnectionError as exception:
    sys.exit(exception.errno)

def setup_options(options):
  '''
  setup options for the rest of the program
  '''
  if not options['directory'].endswith(os.path.sep):
    options['directory'] += os.path.sep
  if not os.path.exists(options['directory']):
    os.makedirs(options['directory'])
  if not os.path.exists(options['directory']):
    os.makedirs(options['directory'])
  if not os.path.exists(options['tables']):
    os.makedirs(options['tables'])
  try:
    if options['from_date']:
      options['from_date'] = datetime.strptime(options['from_date'], '%d-%m-%Y').date()
  except ValueError:
    logging.error('Dates must be in the form dd-mm-YYYY')
    sys.exit(-1)
  options['to_date'] = date.today() + timedelta(days=2)
  options['selection'] = ['High']
  if not options['from_date'] and ('year' not in options or (options['update'])):
    options['from_date'] = date(date.today().year, 1, 1)
  else:
    if options['year'] >= 2010 and options['year'] <= date.today().year:
      options['from_date'] = date(options['year'], 1, 1)
      options['to_date'] = date(options['year'], 12, 31)
  if options['all']:
    options['from_date'] = date(2010, 1, 1)
  if options['from_date'] is None:
    options['from_date'] = options['to_date'] - timedelta(days=7)
  if options['low']:
    options['selection'].append('Low')
  if options['medium']:
    options['selection'].append('Medium')
  if options['unassigned']:
    options['selection'].append('Severity Not Yet Assigned')
  return options

def check_title(tree):
  '''
  check if tree contains vulnerability summary.
  returns a date object if successful.
  '''
  try:
    title = tree.xpath('//title/text()')[0].split(' | ')[0].replace('[\'', '')
  except AttributeError:
    title = ''

  response = re.search(r'Vulnerability Summary for the Week of (\w+\s[0-9]{1,2}\,\s[0-9]{4})', title)
  if response:
    date_object = datetime.strptime(response.group(1), '%B %d, %Y')
    return date_object.date()
  else:
    return False

def make_csv_files(tree, vuln_type, bulletin_name, options):
  '''
  takes selections and grabs html and converts to csv
  '''
  vulnerabilities = list()
  headers = ['Vendor', 'Product', 'Description', 'Published', 'CVSS', 'CVSS Score', 'Source Info']
  vendor = ''
  product = ''
  summary = 'Severity Not Yet Assigned' if vuln_type == 'Severity Not Yet Assigned' else '{0} Vulnerabilities'.format(vuln_type)

  # create vulnerability array
  for vuln in tree.xpath('//table[@summary="{0}"]/tbody/tr'.format(summary)):
    if len(vuln) > 1:
      try:
        try:
          vendor, product = vuln[0].text.split(' -- ')
        except AttributeError:
          # on occasion, US-CERT throws in random
 tags and it screws with the parsing
          vendor, product = vuln[0][0].text.split(' -- ')
      except ValueError:
        vendor = vuln[0].text
      product = vuln[0].text  # Feedback says the prefer the total string in the product column.  Leave as temporary for now.
      description = vuln[1].text
      published = vuln[2].text
      cvss = vuln[4][0].text
      try:
        cvss_score = vuln[3][0].text
      except IndexError:
        cvss_score = '0'

      source_info = build_links(vuln[4], options['link'])
      #source_info = (b' '.join(list(map(lambda x:html.tostring(x), vuln[4])))).decode("utf-8")

      current_vuln = [vendor, product, description, published, cvss, cvss_score, source_info]
      vulnerabilities.append(current_vuln)

  df = pd.DataFrame(vulnerabilities, columns=headers)
  filename = '{0}/{1} - {2} Vulnerabilities.csv'.format(options['tables'], bulletin_name, vuln_type)
  df.to_csv(filename, index=False, encoding='utf-8-sig')

def build_links(element, link_type):
  '''
  build links from source & patch info column
  '''
  source = ''

  if link_type == 'a':
    # loop if requesting anchor tags
    for i in range(0, len(element), 2):
      source_info_href = element[i].get("href")
      source_info_text = element[i].text
      source += '{1}
'.format(source_info_href, source_info_text)
  else:
    # cannot loop as csvs do not support multiple hyperlinks (particularly excel)
    source_info_href = element[0].get("href")
    source_info_text = element[0].text
    source += '=HYPERLINK("{0}","{1}")'.format(source_info_href, source_info_text)
  return source

def parse_arguments():
  '''
  parse command line args, exits on invalid args
  '''
  parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=textwrap.dedent('''\
    Downloads and parses vulnerability summaries from the US-CERT website.
    Creates CSV file(s) for further dissemination.

    Example usage:  python uscert.py -clum --latest --link a

    Copyright (C) 2017 Adam Schaal
    MIT License'''))

  parser.add_argument('-a', '--all', action='store_true', help='retrieve all missing bulletin since 2010 (lengthy!)')
  parser.add_argument('-b', '--bulletin', action='store_true', help='retrieve a specific bulletin')
  parser.add_argument('-c', '--csv', action='store_true', help='creates csv files from html')
  parser.add_argument('-d', '--directory', action='store', default='bulletins', help='name of directory for saving bulletins - default(bulletins)')
  parser.add_argument('-f', '--force', action='store_true', help='force download, ignore / overwrite cached directory')
  parser.add_argument('--from-date', action='store', help='starting date (dd-mm-YYYY)')
  parser.add_argument('-l', '--low', action='store_true', help='select low vulnerabilities as well')
  parser.add_argument('--latest', action='store_true', help='show latest bulletin')
  parser.add_argument('--link', action='store', help='choose "a" for anchor tags or "h" for csv hyperlinks - default(h)')
  parser.add_argument('-m', '--medium', action='store_true', help='show medium vulnerabilities as well')
  parser.add_argument('-t', '--tables', action='store', default='tables', help='name of directory for saving tables - default(tables)')
  parser.add_argument('-u', '--unassigned', action='store_true', help='show Severity Not Yet Assigned vulnerabilities as well')
  parser.add_argument('--update', action='store_true', help='retrieve all newest bulletin since last update')
  parser.add_argument('--year', action='store', nargs='?', default=0, type=int, help='retrieve all bulletins for a given year year')

  return vars(parser.parse_args())

def main():
  '''
  main loop
  '''
  options = parse_arguments()
  setup_options(options)

  for bulletin_name in get_bulletin_list(options):
    if options['bulletin']:
      bulletin_name = options['bulletin']
    filename = '{0}{1}.html'.format(options['directory'], bulletin_name)
    tree = retrieve_bulletin(filename, bulletin_name, options)
    if tree is not None and options['csv']:
      for vuln_type in options['selection']:
        make_csv_files(tree, vuln_type, bulletin_name, options)

if __name__ == "__main__":
	main()<|endoftext|>


<commit_before>from .util.decorators import command, initializer, require_admin
import itertools
import re
import codecs
import traceback
from itertools import zip_longest
import string

string.ascii = [chr(i) for i in range(128)]

SUB_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:s|(.+?)/s)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([gixs]{0,4})?")
TRANS_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:y|(.+?)/y)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([cds]+)?")

def sed(bot, nick, chan, msg):
    if not SUB_REGEX.match(msg) and not TRANS_REGEX.match(msg):
        return # this isn't a sed message

    if SUB_REGEX.match(msg):
        do_sub(bot, nick, chan, msg)
    elif TRANS_REGEX.match(msg):
        tr(bot, nick, chan, msg)

def squeeze(lst, src):
    for ch in lst:
        src = re.sub(ch + r"{2,}", ch, src)
    return src

def sugar(src):
    return src.replace("\\/", "/") \
            .replace("[:upper:]", string.ascii_uppercase) \
            .replace("[:lower:]", string.ascii_lowercase) \
            .replace("[:alpha:]", string.ascii_letters) \
            .replace("[:digit:]", string.digits) \
            .replace("[:xdigit:]", string.hexdigits) \
            .replace("[:alnum:]", string.digits + string.ascii_letters) \
            .replace("[:blank:]", string.whitespace) \
            .replace("[:punct:]", string.punctuation) \
            .replace("[:cntrl:]", "".join([i for i in string.ascii if not i in string.printable])) \
            .replace("[:print:]", string.printable)

def expand(src):
    out = []
    escaped = False
    hyphen = False

    for char in src:
        if char == "\\":
            escaped = True
            continue
        elif char == '-' and not escaped:
            hyphen = True
            escaped = False
            continue
        elif hyphen:
            out.extend(range(out[-1] + 1, ord(char)))
            hyphen = False
        out.append(ord(char))
    return "".join([chr(i) for i in out])

def tr(bot, nick, chan, msg):
    target, qual, lhs, rhs, flags = TRANS_REGEX.match(msg).groups()

    # if there is no qualifier, match the last message from this chan using .+
    target = target or nick

    lhs = expand(sugar(lhs))
    rhs = expand(sugar(rhs))

    qual = qual or '|'.join(list(lhs))

    trans_table = {ord(i[0]):i[1] for i in itertools.zip_longest(lhs, rhs, fillvalue=rhs[-1])}

    msg = get_tr_message(bot, qual, target.lower(), chan)

    msg = msg.translate(trans_table)
    if '\x01' in msg:
        msg = msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (target, msg))
    return bot.msg(chan, "<%s> %s" % (target, msg))

class Substitution(object):
    """ A class representing a substitution in sed. """
    def __init__(self, pattern):
        self.pattern = pattern
        self.target, self.qual, self.re, self.sub, self.flags, self.count = self.parse(pattern)

    def __repr__(self):
        return "Substitution(%r)" % (self.pattern)

    def parse(self, pattern):
        m = SUB_REGEX.match(pattern)
        if not m:
            raise TypeError("Pattern %r isn't a proper substitution pattern!" % (pattern))
        count = 1

        groups = m.groups()
        if 'g' in groups[4]:
            count = 0

        flags = 0
        for c in groups[4]: # for character in flags
            if c != 'g':
                flags += getattr(re, c.upper())

        sub = groups[3].replace("\\/", "/")
        sub = re.sub(r"(?<!\\)(\\)(?=\d+|g<\w+>)", r"\\\\", sub)
        sub = codecs.escape_decode(bytes(sub, "utf-8"))[0].decode('utf-8')

        return list(groups[:-2]) + [sub, flags, count]

    def do(self, on):
        return re.sub(self.re, self.sub, on, count=self.count, flags=self.flags)

def do_sub(bot, nick, chan, msg):
    if !(bot.getconfig()["sed-enabled"]):
        return
    sub = Substitution(msg)

    if sub.target:
        nick = sub.target

    message = get_sub_message(bot, sub, nick.lower(), chan)

    if not message:
        print(nick.lower(), str(sub))
        return

    new_msg = sub.do(message.msg)
    if message.is_action:
        new_msg = new_msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (nick, new_msg))

    return bot.msg(chan, "<%s> %s" % (nick, new_msg))

def get_tr_message(bot, qual, nick, chan):
    if not nick in bot.state.messages:
        return ""

    for message in bot.state.messages[nick]:
        if message.channel != bot.state.channels[chan]:
            continue
        if re.search(qual, message.msg):
            return message.msg
    return ""

def get_sub_message(bot, sub, nick, chan):
    target = nick

    if not target in bot.state.messages:
        return ""

    for message in bot.state.messages[nick]:
        if not message.channel == bot.state.channels[chan]:
            continue
        msg = message.msg
        try:
            if sub.qual:
                if re.search(sub.re, msg) and re.search(sub.qual, msg) and not SUB_REGEX.match(msg):
                    return message
            else:
                if re.search(sub.re, msg) and not SUB_REGEX.match(msg):
                    return message
        except BaseException:
            traceback.print_exc()
    return ""

@initializer
def pluginit(bot):
    if bot.config["sed-enabled"]:
        bot.state.events.MessageEvent.register(sed)
<commit_msg>s/!/not/<commit_after>from .util.decorators import command, initializer, require_admin
import itertools
import re
import codecs
import traceback
from itertools import zip_longest
import string

string.ascii = [chr(i) for i in range(128)]

SUB_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:s|(.+?)/s)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([gixs]{0,4})?")
TRANS_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:y|(.+?)/y)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([cds]+)?")

def sed(bot, nick, chan, msg):
    if not SUB_REGEX.match(msg) and not TRANS_REGEX.match(msg):
        return # this isn't a sed message

    if SUB_REGEX.match(msg):
        do_sub(bot, nick, chan, msg)
    elif TRANS_REGEX.match(msg):
        tr(bot, nick, chan, msg)

def squeeze(lst, src):
    for ch in lst:
        src = re.sub(ch + r"{2,}", ch, src)
    return src

def sugar(src):
    return src.replace("\\/", "/") \
            .replace("[:upper:]", string.ascii_uppercase) \
            .replace("[:lower:]", string.ascii_lowercase) \
            .replace("[:alpha:]", string.ascii_letters) \
            .replace("[:digit:]", string.digits) \
            .replace("[:xdigit:]", string.hexdigits) \
            .replace("[:alnum:]", string.digits + string.ascii_letters) \
            .replace("[:blank:]", string.whitespace) \
            .replace("[:punct:]", string.punctuation) \
            .replace("[:cntrl:]", "".join([i for i in string.ascii if not i in string.printable])) \
            .replace("[:print:]", string.printable)

def expand(src):
    out = []
    escaped = False
    hyphen = False

    for char in src:
        if char == "\\":
            escaped = True
            continue
        elif char == '-' and not escaped:
            hyphen = True
            escaped = False
            continue
        elif hyphen:
            out.extend(range(out[-1] + 1, ord(char)))
            hyphen = False
        out.append(ord(char))
    return "".join([chr(i) for i in out])

def tr(bot, nick, chan, msg):
    target, qual, lhs, rhs, flags = TRANS_REGEX.match(msg).groups()

    # if there is no qualifier, match the last message from this chan using .+
    target = target or nick

    lhs = expand(sugar(lhs))
    rhs = expand(sugar(rhs))

    qual = qual or '|'.join(list(lhs))

    trans_table = {ord(i[0]):i[1] for i in itertools.zip_longest(lhs, rhs, fillvalue=rhs[-1])}

    msg = get_tr_message(bot, qual, target.lower(), chan)

    msg = msg.translate(trans_table)
    if '\x01' in msg:
        msg = msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (target, msg))
    return bot.msg(chan, "<%s> %s" % (target, msg))

class Substitution(object):
    """ A class representing a substitution in sed. """
    def __init__(self, pattern):
        self.pattern = pattern
        self.target, self.qual, self.re, self.sub, self.flags, self.count = self.parse(pattern)

    def __repr__(self):
        return "Substitution(%r)" % (self.pattern)

    def parse(self, pattern):
        m = SUB_REGEX.match(pattern)
        if not m:
            raise TypeError("Pattern %r isn't a proper substitution pattern!" % (pattern))
        count = 1

        groups = m.groups()
        if 'g' in groups[4]:
            count = 0

        flags = 0
        for c in groups[4]: # for character in flags
            if c != 'g':
                flags += getattr(re, c.upper())

        sub = groups[3].replace("\\/", "/")
        sub = re.sub(r"(?<!\\)(\\)(?=\d+|g<\w+>)", r"\\\\", sub)
        sub = codecs.escape_decode(bytes(sub, "utf-8"))[0].decode('utf-8')

        return list(groups[:-2]) + [sub, flags, count]

    def do(self, on):
        return re.sub(self.re, self.sub, on, count=self.count, flags=self.flags)

def do_sub(bot, nick, chan, msg):
    if not (bot.getconfig()["sed-enabled"]):
        return
    sub = Substitution(msg)

    if sub.target:
        nick = sub.target

    message = get_sub_message(bot, sub, nick.lower(), chan)

    if not message:
        print(nick.lower(), str(sub))
        return

    new_msg = sub.do(message.msg)
    if message.is_action:
        new_msg = new_msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (nick, new_msg))

    return bot.msg(chan, "<%s> %s" % (nick, new_msg))

def get_tr_message(bot, qual, nick, chan):
    if not nick in bot.state.messages:
        return ""

    for message in bot.state.messages[nick]:
        if message.channel != bot.state.channels[chan]:
            continue
        if re.search(qual, message.msg):
            return message.msg
    return ""

def get_sub_message(bot, sub, nick, chan):
    target = nick

    if not target in bot.state.messages:
        return ""

    for message in bot.state.messages[nick]:
        if not message.channel == bot.state.channels[chan]:
            continue
        msg = message.msg
        try:
            if sub.qual:
                if re.search(sub.re, msg) and re.search(sub.qual, msg) and not SUB_REGEX.match(msg):
                    return message
            else:
                if re.search(sub.re, msg) and not SUB_REGEX.match(msg):
                    return message
        except BaseException:
            traceback.print_exc()
    return ""

@initializer
def pluginit(bot):
    if bot.config["sed-enabled"]:
        bot.state.events.MessageEvent.register(sed)
<|endoftext|>


<commit_before>#!/usr/bin/python

#============================ adjust path =====================================

import sys
import os

if __name__ == "__main__":
    here = sys.path[0]
    sys.path.insert(0, os.path.join(here, '..', 'sol'))

#============================ imports =========================================

import pickle
import time
import json
import subprocess
import threading
import traceback
from   optparse                 import OptionParser
from   ConfigParser             import SafeConfigParser

import influxdb
import bottle

import OpenCli
import Sol
import SolVersion
import server_version
import flatdict

#============================ defines =========================================

DEFAULT_TCPPORT              = 8081
DEFAULT_SERVERHOST           = '0.0.0.0' # listten on all interfaces

DEFAULT_CONFIGFILE           = 'server.config'
DEFAULT_CRASHLOG             = 'server.crashlog'
DEFAULT_BACKUPFILE           = 'server.backup'
# config file
DEFAULT_SERVERTOKEN          = 'DEFAULT_SERVERTOKEN'
DEFAULT_BASESTATIONTOKEN     = 'DEFAULT_BASESTATIONTOKEN'
DEFAULT_SERVERCERT           = 'server.cert'
DEFAULT_SERVERPRIVKEY        = 'server.ppk'

# stats
STAT_NUM_JSON_REQ            = 'NUM_JSON_REQ'
STAT_NUM_JSON_UNAUTHORIZED   = 'NUM_JSON_UNAUTHORIZED'
STAT_NUM_CRASHES             = 'NUM_CRASHES'
STAT_NUM_OBJECTS_DB_OK       = 'STAT_NUM_OBJECTS_DB_OK'
STAT_NUM_OBJECTS_DB_FAIL     = 'STAT_NUM_OBJECTS_DB_FAIL'

#============================ helpers =========================================

def logCrash(threadName,err):
    output  = []
    output += ["==============================================================="]
    output += [time.strftime("%m/%d/%Y %H:%M:%S UTC",time.gmtime())]
    output += [""]
    output += ["CRASH in Thread {0}!".format(threadName)]
    output += [""]
    output += ["=== exception type ==="]
    output += [str(type(err))]
    output += [""]
    output += ["=== traceback ==="]
    output += [traceback.format_exc()]
    output  = '\n'.join(output)
    # update stats
    AppData().incrStats(STAT_NUM_CRASHES)
    print output
    with open(DEFAULT_CRASHLOG,'a') as f:
        f.write(output)

#============================ classes =========================================

class AppData(object):
    _instance = None
    _init     = False
    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(AppData,cls).__new__(cls, *args, **kwargs)
        return cls._instance
    def __init__(self):
        if self._init:
            return
        self._init      = True
        self.dataLock   = threading.RLock()
        try:
            with open(DEFAULT_BACKUPFILE,'r') as f:
                self.data = pickle.load(f)
        except:
            self.data = {
                'stats' : {},
                'config' : {
                    'servertoken':          DEFAULT_SERVERTOKEN,
                    'basestationtoken':     DEFAULT_BASESTATIONTOKEN,
                },
            }
            self._backupData()
    def incrStats(self,statName,step=1):
        with self.dataLock:
            if statName not in self.data['stats']:
                self.data['stats'][statName] = 0
            self.data['stats'][statName] += step
    def getStats(self):
        with self.dataLock:
            return self.data['stats'].copy()
    def getConfig(self,key):
        with self.dataLock:
            return self.data['config'][key]
    def getAllConfig(self):
        with self.dataLock:
            return self.data['config'].copy()
    def setConfig(self,key,value):
        with self.dataLock:
            self.data['config'][key] = value
        self._backupData()
    def _backupData(self):
        with self.dataLock:
            with open(DEFAULT_BACKUPFILE,'w') as f:
                pickle.dump(self.data,f)

class CherryPySSL(bottle.ServerAdapter):
    def run(self, handler):
        from cherrypy import wsgiserver
        from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter
        server = wsgiserver.CherryPyWSGIServer((self.host, self.port), handler)
        server.ssl_adapter = pyOpenSSLAdapter(
            certificate           = DEFAULT_SERVERCERT,
            private_key           = DEFAULT_SERVERPRIVKEY,
        )
        try:
            server.start()
        finally:
            server.stop()

class Server(threading.Thread):

    def __init__(self,tcpport):

        # store params
        self.tcpport    = tcpport

        # local variables
        AppData()
        self.sol                  = Sol.Sol()
        self.servertoken          = DEFAULT_SERVERTOKEN
        self.basestationtoken     = DEFAULT_BASESTATIONTOKEN
        self.influxClient         = influxdb.client.InfluxDBClient(
                                            host='localhost',
                                            port='8086',
                                            database='realms'
                                        )

        # initialize web server
        self.web        = bottle.Bottle()
        #self.web.route(path='/',                   method='GET', callback=self._cb_root_GET)
        self.web.route(path='/api/v1/echo.json',   method='POST',callback=self._cb_echo_POST)
        self.web.route(path='/api/v1/status.json', method='GET', callback=self._cb_status_GET)
        self.web.route(path='/api/v1/o.json',      method='PUT', callback=self._cb_o_PUT)

        # start the thread
        threading.Thread.__init__(self)
        self.name       = 'Server'
        self.daemon     = True
        self.start()

    def run(self):
        try:
            self.web.run(
                host   = DEFAULT_SERVERHOST,
                port   = self.tcpport,
                server = CherryPySSL,
                quiet  = True,
                debug  = False,
            )

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)

    #======================== public ==========================================

    def close(self):
        # bottle thread is daemon, it will close when main thread closes
        pass

    #======================== private =========================================

    #=== JSON request handler

    def _cb_root_GET(self):
        return 'It works!'

    def _cb_echo_POST(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            bottle.response.content_type = bottle.request.content_type
            return bottle.request.body.read()

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    def _cb_status_GET(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            returnVal = {}
            returnVal['version server']   = server_version.VERSION
            returnVal['version Sol']      = SolVersion.VERSION
            returnVal['uptime computer']  = self._exec_cmd('uptime')
            returnVal['utc']              = int(time.time())
            returnVal['date']             = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime())
            returnVal['last reboot']      = self._exec_cmd('last reboot')
            returnVal['stats']            = AppData().getStats()

            bottle.response.content_type = 'application/json'
            return json.dumps(returnVal)

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    def _cb_o_PUT(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            # abort if malformed JSON body
            if bottle.request.json==None:
                raise bottle.HTTPResponse(
                    body   = json.dumps({'error': 'Malformed JSON body'}),
                    status = 400,
                    headers= {'Content-Type': 'application/json'},
                )

            # parse dicts
            try:
                dicts = self.sol.json_to_dicts(bottle.request.json)
            except:
                raise bottle.HTTPResponse(
                    body   = json.dumps({'error': 'Malformed JSON body contents'}),
                    status = 400,
                    headers= {'Content-Type': 'application/json'},
                )

            # only keep DUST_RAW (sol-type = 14)
            dicts[:] = [obj for obj in dicts if obj['measurement'] == 14]

            # parse objects values
            for obj in dicts:
                obj['fields'] = self.sol.parse_value(obj['measurement'],*obj['fields'])

            # flatten dicts for influxdb
            for obj in dicts:
                obj['fields'] = flatdict.FlatDict(obj['fields'])

            # publish contents
            try:
                self.influxClient.write_points(dicts)
            except:
                AppData().incrStats(STAT_NUM_OBJECTS_DB_FAIL,len(dicts))
                raise
            else:
                AppData().incrStats(STAT_NUM_OBJECTS_DB_OK,len(dicts))

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    #=== misc

    def _authorizeClient(self):
        if bottle.request.headers.get('X-REALMS-Token')!=self.servertoken:
            AppData().incrStats(STAT_NUM_JSON_UNAUTHORIZED)
            raise bottle.HTTPResponse(
                body   = json.dumps({'error': 'Unauthorized'}),
                status = 401,
                headers= {'Content-Type': 'application/json'},
            )

    def _exec_cmd(self,cmd):
        returnVal = None
        try:
            returnVal = subprocess.check_output(cmd, shell=False)
        except:
            returnVal = "ERROR"
        return returnVal

#============================ main ============================================

server = None

def quitCallback():
    global server

    server.close()

def cli_cb_stats(params):
    stats = AppData().getStats()
    output = []
    for k in sorted(stats.keys()):
        output += ['{0:<30}: {1}'.format(k,stats[k])]
    output = '\n'.join(output)
    print output

def main(tcpport):
    global server

    # create the server instance
    server = Server(
        tcpport
    )

    # start the CLI interface
    cli = OpenCli.OpenCli(
        "Server",
        server_version.VERSION,
        quitCallback,
        [
            ("Sol",SolVersion.VERSION),
        ],
    )
    cli.registerCommand(
        'stats',
        's',
        'print the stats',
        [],
        cli_cb_stats
    )

if __name__ == '__main__':
    # parse the config file
    cf_parser = SafeConfigParser()
    cf_parser.read(DEFAULT_CONFIGFILE)

    if cf_parser.has_section('basestation'):
        if cf_parser.has_option('basestation','token'):
            DEFAULT_BASESTATIONTOKEN = cf_parser.get('basestation','token')

    if cf_parser.has_section('server'):
        if cf_parser.has_option('server','host'):
            DEFAULT_SERVER = cf_parser.get('server','host')
        if cf_parser.has_option('server','tcpport'):
            DEFAULT_TCPPORT = cf_parser.getint('server','tcpport')
        if cf_parser.has_option('server','token'):
            DEFAULT_SERVERTOKEN = cf_parser.get('server','token')
        if cf_parser.has_option('server','certfile'):
            DEFAULT_SERVERCERT = cf_parser.get('server','certfile')
        if cf_parser.has_option('server','privatekey'):
            DEFAULT_SERVERPRIVKEY = cf_parser.get('server','privatekey')
        if cf_parser.has_option('server','crashlogfile'):
            DEFAULT_CRASHLOG = cf_parser.get('server','crashlogfile')
        if cf_parser.has_option('server','backupfile'):
            DEFAULT_BACKUPFILE = cf_parser.get('server','backupfile')

    # parse the command line
    parser = OptionParser("usage: %prog [options]")
    parser.add_option(
        "-t", "--tcpport", dest="tcpport",
        default=DEFAULT_TCPPORT,
        help="TCP port to start the JSON API on."
    )
    (options, args) = parser.parse_args()

    main(
        options.tcpport,
    )
<commit_msg>now independently adapted for InfluxDB. No need for Sol to adapt the object format<commit_after>#!/usr/bin/python

#============================ adjust path =====================================

import sys
import os

if __name__ == "__main__":
    here = sys.path[0]
    sys.path.insert(0, os.path.join(here, '..', 'sol'))

#============================ imports =========================================

import pickle
import time
import json
import subprocess
import threading
import traceback
from   optparse                 import OptionParser
from   ConfigParser             import SafeConfigParser

import influxdb
import bottle

import OpenCli
import Sol
import SolVersion
import SolDefines
import server_version
import flatdict
import datetime

#============================ defines =========================================

DEFAULT_TCPPORT              = 8081
DEFAULT_SERVERHOST           = '0.0.0.0' # listten on all interfaces

DEFAULT_CONFIGFILE           = 'server.config'
DEFAULT_CRASHLOG             = 'server.crashlog'
DEFAULT_BACKUPFILE           = 'server.backup'
# config file
DEFAULT_SERVERTOKEN          = 'DEFAULT_SERVERTOKEN'
DEFAULT_BASESTATIONTOKEN     = 'DEFAULT_BASESTATIONTOKEN'
DEFAULT_SERVERCERT           = 'server.cert'
DEFAULT_SERVERPRIVKEY        = 'server.ppk'

# stats
STAT_NUM_JSON_REQ            = 'NUM_JSON_REQ'
STAT_NUM_JSON_UNAUTHORIZED   = 'NUM_JSON_UNAUTHORIZED'
STAT_NUM_CRASHES             = 'NUM_CRASHES'
STAT_NUM_OBJECTS_DB_OK       = 'STAT_NUM_OBJECTS_DB_OK'
STAT_NUM_OBJECTS_DB_FAIL     = 'STAT_NUM_OBJECTS_DB_FAIL'

#============================ helpers =========================================

def logCrash(threadName,err):
    output  = []
    output += ["==============================================================="]
    output += [time.strftime("%m/%d/%Y %H:%M:%S UTC",time.gmtime())]
    output += [""]
    output += ["CRASH in Thread {0}!".format(threadName)]
    output += [""]
    output += ["=== exception type ==="]
    output += [str(type(err))]
    output += [""]
    output += ["=== traceback ==="]
    output += [traceback.format_exc()]
    output  = '\n'.join(output)
    # update stats
    AppData().incrStats(STAT_NUM_CRASHES)
    print output
    with open(DEFAULT_CRASHLOG,'a') as f:
        f.write(output)

def o_to_influx(dicts):
    '''
        Transform list of Sol Objects to list of InfluxDB points
        Args: dicts (list) list of dictionaries
        Returns: idicts (list) list of converted dictionaries
        Exemple:
            dicts = {
                "timestamp" : 1455202067
                "mac" : [ 0, 23, 13, 0, 0, 56, 0, 99 ]
                "type" 14
                "value" : [ 240, 185, 240, 185, 0, 0 ]
            }
    '''
    idicts = []

    for obj in dicts:
        iobj = {}
        iobj['tags'] = {}
        iobj['fields'] = {}

        # (temporary) only keep DUST RAW  and HR
        if (obj['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_DATA_RAW or
            obj['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HR_DEVICE or
            obj['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HR_NEIGHBORS or
            obj['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HR_DISCOVERED):

            # convert timestamp to UTC
            iobj['time'] = datetime.datetime.utcfromtimestamp(obj['timestamp'])

            # change type name
            iobj['measurement'] = obj['type']

            # populate tags
            iobj['tags']['mac'] = obj['mac']

            # populate fields
            iobj['fields'] = flatdict.FlatDict(obj['value'])

            # append element to list
            idicts.append(iobj)

    return idicts

#============================ classes =========================================

class AppData(object):
    _instance = None
    _init     = False
    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(AppData,cls).__new__(cls, *args, **kwargs)
        return cls._instance
    def __init__(self):
        if self._init:
            return
        self._init      = True
        self.dataLock   = threading.RLock()
        try:
            with open(DEFAULT_BACKUPFILE,'r') as f:
                self.data = pickle.load(f)
        except:
            self.data = {
                'stats' : {},
                'config' : {
                    'servertoken':          DEFAULT_SERVERTOKEN,
                    'basestationtoken':     DEFAULT_BASESTATIONTOKEN,
                },
            }
            self._backupData()
    def incrStats(self,statName,step=1):
        with self.dataLock:
            if statName not in self.data['stats']:
                self.data['stats'][statName] = 0
            self.data['stats'][statName] += step
    def getStats(self):
        with self.dataLock:
            return self.data['stats'].copy()
    def getConfig(self,key):
        with self.dataLock:
            return self.data['config'][key]
    def getAllConfig(self):
        with self.dataLock:
            return self.data['config'].copy()
    def setConfig(self,key,value):
        with self.dataLock:
            self.data['config'][key] = value
        self._backupData()
    def _backupData(self):
        with self.dataLock:
            with open(DEFAULT_BACKUPFILE,'w') as f:
                pickle.dump(self.data,f)

class CherryPySSL(bottle.ServerAdapter):
    def run(self, handler):
        from cherrypy import wsgiserver
        from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter
        server = wsgiserver.CherryPyWSGIServer((self.host, self.port), handler)
        server.ssl_adapter = pyOpenSSLAdapter(
            certificate           = DEFAULT_SERVERCERT,
            private_key           = DEFAULT_SERVERPRIVKEY,
        )
        try:
            server.start()
        finally:
            server.stop()

class Server(threading.Thread):

    def __init__(self,tcpport):

        # store params
        self.tcpport    = tcpport

        # local variables
        AppData()
        self.sol                  = Sol.Sol()
        self.servertoken          = DEFAULT_SERVERTOKEN
        self.basestationtoken     = DEFAULT_BASESTATIONTOKEN
        self.influxClient         = influxdb.client.InfluxDBClient(
                                            host='localhost',
                                            port='8086',
                                            database='realms'
                                        )

        # initialize web server
        self.web        = bottle.Bottle()
        #self.web.route(path='/',                   method='GET', callback=self._cb_root_GET)
        self.web.route(path='/api/v1/echo.json',   method='POST',callback=self._cb_echo_POST)
        self.web.route(path='/api/v1/status.json', method='GET', callback=self._cb_status_GET)
        self.web.route(path='/api/v1/o.json',      method='PUT', callback=self._cb_o_PUT)

        # start the thread
        threading.Thread.__init__(self)
        self.name       = 'Server'
        self.daemon     = True
        self.start()

    def run(self):
        try:
            self.web.run(
                host   = DEFAULT_SERVERHOST,
                port   = self.tcpport,
                server = CherryPySSL,
                quiet  = True,
                debug  = False,
            )

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)

    #======================== public ==========================================

    def close(self):
        # bottle thread is daemon, it will close when main thread closes
        pass

    #======================== private =========================================

    #=== JSON request handler

    def _cb_root_GET(self):
        return 'It works!'

    def _cb_echo_POST(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            bottle.response.content_type = bottle.request.content_type
            return bottle.request.body.read()

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    def _cb_status_GET(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            returnVal = {}
            returnVal['version server']   = server_version.VERSION
            returnVal['version Sol']      = SolVersion.VERSION
            returnVal['uptime computer']  = self._exec_cmd('uptime')
            returnVal['utc']              = int(time.time())
            returnVal['date']             = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime())
            returnVal['last reboot']      = self._exec_cmd('last reboot')
            returnVal['stats']            = AppData().getStats()

            bottle.response.content_type = 'application/json'
            return json.dumps(returnVal)

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    def _cb_o_PUT(self):
        try:
            # update stats
            AppData().incrStats(STAT_NUM_JSON_REQ)

            # authorize the client
            self._authorizeClient()

            # abort if malformed JSON body
            if bottle.request.json==None:
                raise bottle.HTTPResponse(
                    body   = json.dumps({'error': 'Malformed JSON body'}),
                    status = 400,
                    headers= {'Content-Type': 'application/json'},
                )

            # parse dicts
            try:
                dicts = self.sol.json_to_dicts(bottle.request.json)
            except:
                raise bottle.HTTPResponse(
                    body   = json.dumps({'error': 'Malformed JSON body contents'}),
                    status = 400,
                    headers= {'Content-Type': 'application/json'},
                )

            # parse objects values
            for obj in dicts:
                obj['value'] = self.sol.parse_value(obj['type'],*obj['value'])

            # transform Sol Objects into InfluxDB points
            idicts = o_to_influx(dicts)
            print idicts

            # publish contents
            try:
                self.influxClient.write_points(idicts)
            except:
                AppData().incrStats(STAT_NUM_OBJECTS_DB_FAIL,len(dicts))
                raise
            else:
                AppData().incrStats(STAT_NUM_OBJECTS_DB_OK,len(dicts))

        except bottle.BottleException:
            raise

        except Exception as err:
            logCrash(self.name,err)
            raise

    #=== misc

    def _authorizeClient(self):
        if bottle.request.headers.get('X-REALMS-Token')!=self.servertoken:
            AppData().incrStats(STAT_NUM_JSON_UNAUTHORIZED)
            raise bottle.HTTPResponse(
                body   = json.dumps({'error': 'Unauthorized'}),
                status = 401,
                headers= {'Content-Type': 'application/json'},
            )

    def _exec_cmd(self,cmd):
        returnVal = None
        try:
            returnVal = subprocess.check_output(cmd, shell=False)
        except:
            returnVal = "ERROR"
        return returnVal

#============================ main ============================================

server = None

def quitCallback():
    global server

    server.close()

def cli_cb_stats(params):
    stats = AppData().getStats()
    output = []
    for k in sorted(stats.keys()):
        output += ['{0:<30}: {1}'.format(k,stats[k])]
    output = '\n'.join(output)
    print output

def main(tcpport):
    global server

    # create the server instance
    server = Server(
        tcpport
    )

    # start the CLI interface
    cli = OpenCli.OpenCli(
        "Server",
        server_version.VERSION,
        quitCallback,
        [
            ("Sol",SolVersion.VERSION),
        ],
    )
    cli.registerCommand(
        'stats',
        's',
        'print the stats',
        [],
        cli_cb_stats
    )

if __name__ == '__main__':
    # parse the config file
    cf_parser = SafeConfigParser()
    cf_parser.read(DEFAULT_CONFIGFILE)

    if cf_parser.has_section('basestation'):
        if cf_parser.has_option('basestation','token'):
            DEFAULT_BASESTATIONTOKEN = cf_parser.get('basestation','token')

    if cf_parser.has_section('server'):
        if cf_parser.has_option('server','host'):
            DEFAULT_SERVER = cf_parser.get('server','host')
        if cf_parser.has_option('server','tcpport'):
            DEFAULT_TCPPORT = cf_parser.getint('server','tcpport')
        if cf_parser.has_option('server','token'):
            DEFAULT_SERVERTOKEN = cf_parser.get('server','token')
        if cf_parser.has_option('server','certfile'):
            DEFAULT_SERVERCERT = cf_parser.get('server','certfile')
        if cf_parser.has_option('server','privatekey'):
            DEFAULT_SERVERPRIVKEY = cf_parser.get('server','privatekey')
        if cf_parser.has_option('server','crashlogfile'):
            DEFAULT_CRASHLOG = cf_parser.get('server','crashlogfile')
        if cf_parser.has_option('server','backupfile'):
            DEFAULT_BACKUPFILE = cf_parser.get('server','backupfile')

    # parse the command line
    parser = OptionParser("usage: %prog [options]")
    parser.add_option(
        "-t", "--tcpport", dest="tcpport",
        default=DEFAULT_TCPPORT,
        help="TCP port to start the JSON API on."
    )
    (options, args) = parser.parse_args()

    main(
        options.tcpport,
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python3.3
from utils import initializer
from yaml import safe_load
from networkx import connected_watts_strogatz_graph
import Pyro4
import random

class Gamer:
    """
    The player
    """
    @initializer
    def __init__(self, name, items, room):
        """
        Da init
        """
        pass
    def pick_up(self):
        pass

    def drop(self):
        pass

class Room:
    """
    The room is on fire(maybe)
    """
    @initializer
    def __init__(self, name, description, items):
        """
        Da init2
        """
        pass

class Item:
    """
    For example bucket, cup or sandwich
    """
    @initializer
    def __init__(self, name, description, capacity=None):
        self.capacity = capacity

if __name__ == '__main__':
    with open("rooms.yaml") as data_file:
        rooms_data = safe_load(data_file)
    room_number = len(rooms_data)
    room_order = list(range(room_number))
    random.shuffle(room_order)
    print(room_order)
    graph = connected_watts_strogatz_graph(5, 2,0.2) #so called small world graph
    print(rooms_data)
    print(Room(1, 3, 4).description)
    print(Item(1, 2).capacity)
<commit_msg>Method implementation<commit_after>#!/usr/bin/env python3.3
from utils import initializer
from yaml import safe_load
from networkx import connected_watts_strogatz_graph
import Pyro4
import random

class Gamer:
    """
    The player
    """
    @initializer
    def __init__(self, name, room, items = []):
        """
        Da init
        """
        self.items = items
    def pick_up(self, item):
        self.items.append(item)

    def drop(self, index):
        self.items.pop(index)

class Room:
    """
    The room is on fire(maybe)
    """
    @initializer
    def __init__(self, name, description, items = []):
        """
        Da init2
        """
        self.items = items

class Item:
    """
    For example bucket, cup or sandwich
    """
    @initializer
    def __init__(self, name, description, capacity=None):
        self.capacity = capacity

FIRE = 1000

if __name__ == '__main__':
    with open("rooms.yaml") as data_file:
        rooms_data = safe_load(data_file)
    room_number = len(rooms_data)
    room_order = list(range(room_number))
    random.shuffle(room_order)
    print(room_order)
    graph = connected_watts_strogatz_graph(room_number+1, 2,0.2) #so called small world graph
    print(rooms_data)
    print(Room(1, 3, 4).description)
    print(Item(1, 2).capacity)
<|endoftext|>


<commit_before>from flask import Flask, url_for, render_template
app = Flask(__name__)

@app.route('/')
def hello_world():
    return render_template('index.html')

# def index(): pass

@app.route('/login')
def login(): pass

@app.route('/user/<username>')
def profile(username): pass

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

# with app.test_request_context():
# 	print (url_for('hello_world'))
# 	print (url_for('login'))
# 	print (url_for('login', next='/', other_param='hah'))
# 	print (url_for('profile', username='John Doe'))

if __name__ == '__main__':
    app.run(debug=True)
    # to make externally visible:
    # app.run(host='0.0.0.0')
<commit_msg>Remove commented sections<commit_after>from flask import Flask, url_for, render_template
app = Flask(__name__)

@app.route('/')
def hello_world():
    return render_template('index.html')

# def index(): pass

@app.route('/login')
def login(): pass

@app.route('/user/<username>')
def profile(username): pass

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

if __name__ == '__main__':
    app.run(debug=True)
    # to make externally visible:
    # app.run(host='0.0.0.0')
<|endoftext|>


<commit_before>""""

BUGS

TODO

Remove hardcoded 'geboekt', 'obligo', etc. from all model/controller - grep
    - model.orders maken die dict retourned met order name from mysql and excel import

Graph
    - graph groups should be named: <ordergroup_file_name><ordergroup_name>

Model
    - model.users
        * ouser lijst gebruiken om menu te bouwen (ordergroepen voor report!)

    - regels
        * write validator for input that can be re-used (tablename in config[''] etc.)
        * add reserve to db

    - orderlist
        * add (merge it with regels)
        * make model.budget.orderlist class that behavious similar to other budget classes
        * sort orderlist on act. code and show them in different collapsoable tables.

Config
    - config module maken waar ook alle data in zit en users
    - alle config params naar db verplaatsen -> model

Controller & Webpages
    - Alle templates doorlopen op $:xxx als dat niet nodig is.
    - Navbar is build on 2 recursive functions. Replace by ordergroup.list_groups(Inverted=True)
      That function should give you the exact list that you need.

    - view
        * html_tree has double code, lot of it is done by budget.kostensoortgroup

    - admin
        * forms layout doen (attributen van webpy forms setten)
        * create test that looks if there are orders in db not in any ordergroup
          (and perhaps also look at the GS)
        * add 'year' field in table for salaris geboekt based on the boekingsdate.

    - login
        * bij password verkeerd invullen post form leegmaken.

    - salaris
        * settings maken
        * Plaatjes maken/koppelen

    - report
        * aantal voorgaande jaren in settings

    -graph
        * userHash checken - alles staat nu op

# TIPS
    - render.<template>(arg1, arg2, arg3, cache=False) will reload the template file everytime you refresh
"""
import web
web.config.debug = True  # Set to False for no ouput! Must be done before the rest
from controller import Index, Report, Admin, Login, Logout, Graph, View, Salaris, Orderlist

### Url mappings
urls = (
    '/view/(.+)', 'View',
    '/login/(.+)', 'Login',
    '/logout/(.+)', 'Logout',
    '/report/(.+)', 'Report',
    '/salaris/(.+)', 'Salaris',
    '/orderlist/(.+)', 'Orderlist',
    '/admin/(.+)', 'Admin',
    '/graph/(.+)/(.+)/(.+)/(.+).png', 'Graph',
    '/index/(.+)', 'Index',
)

app = web.application(urls, globals())
if web.config.get('_session') is None:
    session = web.session.Session(app, web.session.DiskStore('sessions'), {'count': 0})
    web.config._session = session
else:
    session = web.config._session

if __name__ == "__main__":
    app.run()
<commit_msg>adds apache wsgi support for embedded mode<commit_after>""""
BUGS

TODO

Remove hardcoded 'geboekt', 'obligo', etc. from all model/controller - grep
    - model.orders maken die dict retourned met order name from mysql and excel import

Graph
    - graph groups should be named: <ordergroup_file_name><ordergroup_name>

Model
    - model.users
        * ouser lijst gebruiken om menu te bouwen (ordergroepen voor report!)

    - regels
        * write validator for input that can be re-used (tablename in config[''] etc.)
        * add reserve to db

    - orderlist
        * add (merge it with regels)
        * make model.budget.orderlist class that behavious similar to other budget classes
        * sort orderlist on act. code and show them in different collapsoable tables.

Config
    - config module maken waar ook alle data in zit en users
    - alle config params naar db verplaatsen -> model

Controller & Webpages
    - Alle templates doorlopen op $:xxx als dat niet nodig is.
    - Navbar is build on 2 recursive functions. Replace by ordergroup.list_groups(Inverted=True)
      That function should give you the exact list that you need.

    - view
        * html_tree has double code, lot of it is done by budget.kostensoortgroup

    - admin
        * forms layout doen (attributen van webpy forms setten)
        * create test that looks if there are orders in db not in any ordergroup
          (and perhaps also look at the GS)
        * add 'year' field in table for salaris geboekt based on the boekingsdate.

    - login
        * bij password verkeerd invullen post form leegmaken.

    - salaris
        * settings maken
        * Plaatjes maken/koppelen

    - report
        * aantal voorgaande jaren in settings

    -graph
        * userHash checken - alles staat nu op

# TIPS
    - render.<template>(arg1, arg2, arg3, cache=False) will reload the template file everytime you refresh

"""
import sys,os
import web
web.config.debug = True  # Set to False for no ouput! Must be done before the rest

# Apache WSGI requires path and working dir to be changed for importing own modules
app_path= os.path.dirname(__file__)
sys.path.append(app_path)
if app_path:
    os.chdir(app_path)

from controller import Index, Report, Admin, Login, Logout, Graph, View, Salaris, Orderlist

urls = (
    '/view/(.+)', 'View',
    '/login/(.+)', 'Login',
    '/logout/(.+)', 'Logout',
    '/report/(.+)', 'Report',
    '/salaris/(.+)', 'Salaris',
    '/orderlist/(.+)', 'Orderlist',
    '/admin/(.+)', 'Admin',
    '/graph/(.+)/(.+)/(.+)/(.+).png', 'Graph',
    '/index/(.+)', 'Index',
)

# with WSGI:
app = web.application(urls, globals())
if web.config.get('_session') is None:
    session = web.session.Session(app, web.session.DiskStore('sessions'), {'count': 0})
    web.config._session = session
else:
    session = web.config._session
application = app.wsgifunc()

# withouth WSGI
if __name__ == "__main__":
    app.run()
<|endoftext|>