<commit_before>#-------------------------------------------------------------------------------
#
# Controls shed weather station
#
# The MIT License (MIT)
#
# Copyright (c) 2015 William De Freitas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#-------------------------------------------------------------------------------

#!usr/bin/env python

#===============================================================================
# Import modules
#===============================================================================
# Standard Library
import sys
import datetime
import time
import collections

# Third party modules

# Application modules
import thingspeak
import rrd_tools
import settings as s
import log

#===============================================================================
# MAIN
#===============================================================================
def main():

    '''Synchronizes thingspeak account with the data in the local Round Robin
        database.

        Starts by grabing some data from thingspeak to check all sensors are
        present.

        Then checks all sensors are also present in the RRD database.

        If any of these checks fail, the script ends with an error.

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '/home/pi/weather/logs/rrd_ts_sync.log')

    logger.info('--- RRD To Thingspeak Sync Script Started ---')

    try:
        #-----------------------------------------------------------------------
        # SET UP THINGSPEAK ACCOUNT
        #-----------------------------------------------------------------------
        ts_acc = thingspeak.TSChannel(s.THINGSPEAK_HOST_ADDR,
                                      file= s.THINGSPEAK_API_KEY_FILENAME,
                                      ch_id= s.THINGSPEAK_CHANNEL_ID)

        ch_feed = ts_acc.get_a_channel_field_feed(field_id= 1,
                                                parameters= {'results': 1})

        sensor_list = {}

        #Checks each sensor is present in thignspeak and if it is create a dict
        #with the field location of the sensor
        for key in s.SENSOR_SET.keys():
            if key.replace('_', ' ') not in ch_feed["channel"].values():
                logger.error('Data sources in the set up not present in thingspeak fields.')
                logger.error(ch_feed["channel"].values())
                logger.error(s.SENSOR_SET.keys())
                logger.error('Exiting...')
                sys.exit()
            else:
                sensor_list[key] = ch_feed["channel"].keys()[ch_feed["channel"].values().index(key.replace('_', ' '))]

        logger.info(sensor_list)

        logger.info('Thingspeak fetch successful.')

        #-----------------------------------------------------------------------
        # CHECK RRD FILE
        #-----------------------------------------------------------------------
        rrd = rrd_tools.RrdFile(s.RRDTOOL_RRD_FILE)

        if sorted(rrd.ds_list()) != sorted(list(s.SENSOR_SET.keys())):
            logger.error('Data sources in RRD file does not match set up.')
            logger.error(rrd.ds_list())
            logger.error(list(s.SENSOR_SET.keys()))
            logger.error('Exiting...')
            sys.exit()
        else:
            logger.info('RRD fetch successful.')

        #-----------------------------------------------------------------------
        # Get last feed update from thingspeak and RRD
        #-----------------------------------------------------------------------
        last_ts_feed = ts_acc.get_last_entry_in_channel_feed()
        last_rrd_feed = rrd.last_update()

        logger.debug('Last TS feed data:')
        logger.debug(last_ts_feed)

        #Thingspeak returns a -1 if there are no records
        if last_ts_feed == '-1':
            last_ts_feed = last_rrd_feed
        else:
            last_ts_feed = datetime.datetime.strptime(last_ts_feed['created_at'], '%Y-%m-%dT%H:%M:%SZ')
            last_ts_feed = int(time.mktime(last_ts_feed.utctimetuple()))

        logger.info('Last TS feed:  {last_ts_feed_time} {long_time}'.format(
            last_ts_feed_time= last_ts_feed,
            long_time= time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(last_ts_feed))))

        logger.info('Last RRD feed: {last_ts_feed_time} {long_time}'.format(
            last_ts_feed_time= last_rrd_feed,
            long_time= time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(last_rrd_feed))))

        #-----------------------------------------------------------------------
        # Fetch values from rrd
        #-----------------------------------------------------------------------
        rrd_entry = rrd.fetch(start= last_ts_feed - s.UPDATE_RATE)

        rt = collections.namedtuple( 'rt', 'start end step ds value')
        rrd_entry = rt(rrd_entry[0][0], rrd_entry[0][1], rrd_entry[0][2],
                        rrd_entry[1], rrd_entry[2])

        logger.info('Number of entries to update: {feeds}'.format(
                            feeds= len(rrd_entry.value)-2))

        #-----------------------------------------------------------------------
        # Create a list with new thingspeak updates and send it to TS
        #-----------------------------------------------------------------------
        for i in range(1, len(rrd_entry.value)-1):
            next_entry_time = rrd_entry.start + ((i + 1) * s.UPDATE_RATE)

            send_list = {'created_at': '{time}'.format(
                            time=time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(next_entry_time))),
                         'timezone':'Etc/UTC'}

            for key in sorted(s.SENSOR_SET.keys()):
                value = rrd_entry.value[i][rrd_entry.ds.index(key)]
                if value is not None:
                    send_list[sensor_list[key]] = str(value)

            logger.info(send_list)
            response = ts_acc.update_channel(send_list)
            logger.info('Thingspeak update: {reason}'.format(reason= response.reason))

            #Thingspeak update rate is limited to 15s per entry
            time.sleep(20)

            if response.status_code is not 200:
                response = ts_acc.update_channel(send_list)
                logger.error('Retry: {reason}'.format(reason= response.reason))
                time.sleep(20)

        logger.info('--- Script Finished ---')

    except Exception, e:
        logger.error('Update failed ({error_v}). Exiting...'.format(
            error_v=e), exc_info=True)
        sys.exit()

#===============================================================================
# Boiler plate
#===============================================================================
if __name__=='__main__':
    main()
<commit_msg>Rrd_ts_sync factored<commit_after>#-------------------------------------------------------------------------------
#
# Controls shed weather station
#
# The MIT License (MIT)
#
# Copyright (c) 2015 William De Freitas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#-------------------------------------------------------------------------------

#!usr/bin/env python

#===============================================================================
# Import modules
#===============================================================================
# Standard Library
import sys
import datetime
import time
import collections

# Third party modules

# Application modules
import thingspeak
import rrd_tools
import settings as s
import log

#===============================================================================
# SYNC
#===============================================================================
def sync(ts_host, ts_filename, ts_channel_id, sensors, update, rrd_file):

    '''
    Synchronizes thingspeak account with the data in the local Round Robin
        database.

    Starts by grabing some data from thingspeak to check all sensors are
        present.

    Then checks all sensors are also present in the RRD database.

    If any of these checks fail, the script ends with an error.

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '/home/pi/weather/logs/rrd_ts_sync.log')

    logger.info('--- RRD To Thingspeak Sync Script Started ---')

    try:
        #-----------------------------------------------------------------------
        # SET UP THINGSPEAK ACCOUNT
        #-----------------------------------------------------------------------
        ts_acc = thingspeak.TSChannel(ts_host, file= ts_filename, ch_id= ts_channel_id)

        ch_feed = ts_acc.get_a_channel_field_feed(field_id= 1,
                                                parameters= {'results': 1})

        sensor_to_field = {}

        #Checks each sensor is present in thingspeak and if it is create a dict
        #with the field location of the sensor
        for sensor in sensors:
            if sensor.replace('_', ' ') not in ch_feed["channel"].values():
                logger.error('Data sources in the set up not present in thingspeak fields.')
                logger.error(ch_feed["channel"].values())
                logger.error(sensors)
                logger.error('Exiting...')
                sys.exit()
            else:
                sensor_to_field[sensor] = ch_feed["channel"].keys()[ch_feed["channel"].values().index(key.replace('_', ' '))]

        logger.info(sensor_to_field)
        logger.info('Thingspeak fetch successful.')

        #-----------------------------------------------------------------------
        # CHECK RRD FILE
        #-----------------------------------------------------------------------
        rrd = rrd_tools.RrdFile(rrd_file)

        if sorted(rrd.ds_list()) != sorted(sensors):
            logger.error('Data sources in RRD file does not match set up.')
            logger.error(rrd.ds_list())
            logger.error(sensors)
            logger.error('Exiting...')
            sys.exit()
        else:
            logger.info('RRD fetch successful.')

        #-----------------------------------------------------------------------
        # Get last feed update from thingspeak and RRD
        #-----------------------------------------------------------------------
        last_ts_feed = ts_acc.get_last_entry_in_channel_feed()
        last_rrd_feed = rrd.last_update()

        logger.debug('Last TS feed data:')
        logger.debug(last_ts_feed)

        #Thingspeak returns a -1 if there are no records
        if last_ts_feed == '-1':
            last_ts_feed = last_rrd_feed
        else:
            last_ts_feed = datetime.datetime.strptime(last_ts_feed['created_at'], '%Y-%m-%dT%H:%M:%SZ')
            last_ts_feed = int(time.mktime(last_ts_feed.utctimetuple()))

        logger.info('Last TS feed:  {last_ts_feed_time} {long_time}'.format(
            last_ts_feed_time= last_ts_feed,
            long_time= time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(last_ts_feed))))

        logger.info('Last RRD feed: {last_ts_feed_time} {long_time}'.format(
            last_ts_feed_time= last_rrd_feed,
            long_time= time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(last_rrd_feed))))

        #-----------------------------------------------------------------------
        # Fetch values from rrd
        #-----------------------------------------------------------------------
        rrd_entry = rrd.fetch(start= last_ts_feed - update)

        rt = collections.namedtuple( 'rt', 'start end step ds value')
        rrd_entry = rt(rrd_entry[0][0], rrd_entry[0][1], rrd_entry[0][2],
                        rrd_entry[1], rrd_entry[2])

        logger.info('Number of entries to update: {feeds}'.format(
                            feeds= len(rrd_entry.value)-2))

        #-----------------------------------------------------------------------
        # Create a list with new thingspeak updates and send it to TS
        #-----------------------------------------------------------------------
        for i in range(1, len(rrd_entry.value)-1):
            next_entry_time = rrd_entry.start + ((i + 1) * update)

            tx_data = {'created_at': '{time}'.format(
                            time=time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(next_entry_time))),
                         'timezone':'Etc/UTC'}

            #Transmit data to be in the format {'field1': '18.232'}
            for sensor in sorted(sensors):
                value = rrd_entry.value[i][rrd_entry.ds.index(sensor)]
                if value is not None:
                    tx_data[sensor_to_field[sensor]] = str(value)

            logger.info(tx_data)
            response = ts_acc.update_channel(tx_data)
            logger.info('Thingspeak update: {reason}'.format(reason= response.reason))

            #Thingspeak update rate is limited to 15s per entry
            time.sleep(20)

            if response.status_code is not 200:
                response = ts_acc.update_channel(tx_data)
                logger.error('Retry: {reason}'.format(reason= response.reason))
                time.sleep(20)

        logger.info('--- Script Finished ---')

    except Exception, e:
        logger.error('Update failed ({error_v}). Exiting...'.format(
            error_v=e), exc_info=True)
        sys.exit()

#===============================================================================
# MAIN
#===============================================================================
def main():
    sync(s.THINGSPEAK_HOST_ADDR, s.THINGSPEAK_API_KEY_FILENAME, s.THINGSPEAK_CHANNEL_ID,
         list(s.SENSOR_SET.keys()), s.UPDATE_RATE, s.RRDTOOL_RRD_FILE)

#===============================================================================
# Boiler plate
#===============================================================================
if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import argparse
import subprocess as sub

### Parse command line arguments
parser = argparse.ArgumentParser(description="M/M/1 queue simulation -- Helper script")
parser.add_argument('reps', metavar='repetitions',
                    type=int, help='number of repetitions')
parser.add_argument('sim_duration', metavar='simulation_duration',
                    type=int, help='duration of each simulation stage in seconds')
parser.add_argument('int_rate', metavar='interarrival_rate',
                    type=int, help='mean packet interarrival rate in seconds')
parser.add_argument('sr_rate', metavar='service_rate',
                    type=int, help='mean packet service rate in seconds')
parser.add_argument('--batch_size', dest='batch_size', default=4,
                    type=int, help='batch size for multiprocessing')
parser.add_argument('--initial_seed', dest='init_seed', default=0,
                    type=int, help='base for seed values')
args = parser.parse_args()
repetitions = args.reps
sim_duration = args.sim_duration
interarrival_rate = args.int_rate
service_rate = args.sr_rate
batch_size = args.batch_size
init_seed = args.init_seed

### Run simulations
try:
  # One process at a time
  if batch_size == 1:
    for n in range(repetitions):
      sub.call("python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n), shell=True)
  # In batches
  else:
    # Split num of repetitions into batches
    quotient = repetitions // batch_size
    remainder = repetitions % batch_size
    # Run the simulations in parallel as subprocesses
    num_proc = batch_size if batch_size <= repetitions else remainder
    procs = [sub.Popen("python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n), shell=True) for n in range(num_proc)]
    while True:
      procs_poll = list(map(lambda x: x.poll() != None, procs))
      if not all(procs_poll):
        procs[procs_poll.index(False)].wait()
      elif num_proc < repetitions:
        temp_num = batch_size if num_proc + batch_size <= repetitions else remainder
        for n in range(num_proc, num_proc + temp_num):
          procs += [sub.Popen("python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n), shell=True)]
        num_proc += temp_num
      else:
        break
except OSError as e:
  print("Execution failed: ", e)
<commit_msg>Make run-example more readable.<commit_after>#!/usr/bin/env python
# encoding: utf-8

import argparse
import subprocess as sub

### Parse command line arguments
parser = argparse.ArgumentParser(description="M/M/1 queue simulation -- Helper script")
parser.add_argument('reps', metavar='repetitions',
                    type=int, help='number of repetitions')
parser.add_argument('sim_duration', metavar='simulation_duration',
                    type=int, help='duration of each simulation stage in seconds')
parser.add_argument('int_rate', metavar='interarrival_rate',
                    type=int, help='mean packet interarrival rate in seconds')
parser.add_argument('sr_rate', metavar='service_rate',
                    type=int, help='mean packet service rate in seconds')
parser.add_argument('--batch_size', dest='batch_size', default=4,
                    type=int, help='batch size for multiprocessing')
parser.add_argument('--initial_seed', dest='init_seed', default=0,
                    type=int, help='base for seed values')
args = parser.parse_args()
repetitions = args.reps
sim_duration = args.sim_duration
interarrival_rate = args.int_rate
service_rate = args.sr_rate
batch_size = args.batch_size
init_seed = args.init_seed

### Run simulations
try:
  # One process at a time
  if batch_size == 1:
    for n in range(repetitions):
      cmd = "python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n)
      sub.call(cmd, shell=True)
  # In batches
  else:
    # Split num of repetitions into batches
    quotient = repetitions // batch_size
    remainder = repetitions % batch_size
    # Run the simulations in parallel as subprocesses
    num_proc = batch_size if batch_size <= repetitions else remainder
    procs = []
    for n in range(num_proc):
      cmd = "python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n)
      procs += [sub.Popen(cmd, shell=True)]
    while True:
      procs_poll = list(map(lambda x: x.poll() != None, procs))
      if not all(procs_poll):
        procs[procs_poll.index(False)].wait()
      elif num_proc < repetitions:
        temp_num = batch_size if num_proc + batch_size <= repetitions else remainder
        for n in range(num_proc, num_proc + temp_num):
          cmd = "python main-example.py {} {} {} --seed={} --id={}".format(sim_duration, interarrival_rate, service_rate, n+init_seed, n)
          procs += [sub.Popen(cmd, shell=True)]
        num_proc += temp_num
      else:
        break
except OSError as e:
  print("Execution failed: ", e)
<|endoftext|>


<commit_before>from pipeline.pipeline import *
from pipeline.entitylinker import *
from pipeline.triplealigner import *
from pipeline.datareader import DBpediaAbstractsDataReader
from pipeline.writer import *
from pipeline.coreference import *
from utils.triplereader import *
from utils.triplereaderitems import *
from utils.triplereadertriples import *
from utils.labelreader import *
from pipeline.filter import *

# Reading the DBpedia Abstracts Dataset
reader = DBpediaAbstractsDataReader('./datasets/wikipedia-abstracts/csv/sample-dbpedia-abstracts-eo.csv', db_wd_mapping='./datasets/wikidata/sample-dbpedia-wikidata-sameas.csv')

trip_read = TripleReader('./datasets/wikidata/sample-wikidata-triples.csv')
label_read = LabelReader('./datasets/wikidata/sample-wikidata-labels.csv', 'eo')
trip_read_items = TripleReaderItems('./datasets/wikidata/sample-wikidata-triples.csv')
trip_read_trip = TripleReaderTriples('./datasets/wikidata/sample-wikidata-triples.csv')

keyword_ent_linker = KeywordMatchingEntityLinker(trip_read_items, label_read)
Salign = SimpleAligner(trip_read)

#prop = WikidataPropertyLinker('./datasets/wikidata/wikidata-properties.csv')
date = DateLinker()
#SPOalign = SPOAligner(trip_read)
NSalign = NoSubjectAlign(trip_read)
Noalign = NoAligner(trip_read_trip)

fist_sen_lim = FistSentenceLimiter()
main_ent_lim = MainEntityLimiter()

writer = JsonWriter('./out-test', "", 1)

writer_triples = CustomeWriterTriples('./out-test', "", 1)
writer_entities = CustomeWriterEntities('./out-test', "", 1)

for d in reader.read_documents():
	#print d.title

	#print label_read.get(d.docid)
#    try:
	d = keyword_ent_linker.run(d)
	d = date.run(d)
		#d = link.run(d)
	d = NSalign.run(d)
		#d = coref.run(d)
		#d = date.run(d)
	d = Salign.run(d)
		#d = prop.run(d)
		#d = SPOalign.run(d)
	d = fist_sen_lim.run(d)

	if not main_ent_lim.run(d):
		continue

	d = Noalign.run(d)

	writer_triples.run(d)
	writer_entities.run(d)
	writer.run(d)
	print "Document Title: %s \t Number of Annotated Entities %s \t Number of Annotated Triples %s" % (d.title, len(d.entities), len(d.triples))

 #   except Exception as e:

  #      print "error Processing document %s" % d.title
<commit_msg>fix naming convention<commit_after>from pipeline.pipeline import *
from pipeline.entitylinker import *
from pipeline.triplealigner import *
from pipeline.datareader import DBpediaAbstractsDataReader
from pipeline.writer import *
from pipeline.coreference import *
from utils.triplereader import *
from utils.triplereaderitems import *
from utils.triplereadertriples import *
from utils.labelreader import *
from pipeline.filter import *

# Reading the DBpedia Abstracts Dataset
reader = DBpediaAbstractsDataReader('./datasets/wikipedia-abstracts/csv/sample-dbpedia-abstracts-eo.csv', db_wd_mapping='./datasets/wikidata/sample-dbpedia-wikidata-sameas.csv')

trip_read = TripleReader('./datasets/wikidata/sample-wikidata-triples.csv')
label_read = LabelReader('./datasets/wikidata/sample-wikidata-labels.csv', 'eo')
trip_read_items = TripleReaderItems('./datasets/wikidata/sample-wikidata-triples.csv')
trip_read_trip = TripleReaderTriples('./datasets/wikidata/sample-wikidata-triples.csv')

keyword_ent_linker = KeywordMatchingEntityLinker(trip_read_items, label_read)
salign = SimpleAligner(trip_read)

#prop = WikidataPropertyLinker('./datasets/wikidata/wikidata-properties.csv')
date = DateLinker()
#SPOalign = SPOAligner(trip_read)
nsalign = NoSubjectAlign(trip_read)
noalign = NoAligner(trip_read_trip)

fist_sen_lim = FistSentenceLimiter()
main_ent_lim = MainEntityLimiter()

writer = JsonWriter('./out-test', "", 1)

writer_triples = CustomeWriterTriples('./out-test', "", 1)
writer_entities = CustomeWriterEntities('./out-test', "", 1)

for d in reader.read_documents():
	#print d.title

	#print label_read.get(d.docid)
#    try:
	d = keyword_ent_linker.run(d)
	d = date.run(d)
		#d = link.run(d)
	d = nsalign.run(d)
		#d = coref.run(d)
		#d = date.run(d)
	d = salign.run(d)
		#d = prop.run(d)
		#d = SPOalign.run(d)
	d = fist_sen_lim.run(d)

	if not main_ent_lim.run(d):
		continue

	d = noalign.run(d)

	writer_triples.run(d)
	writer_entities.run(d)
	writer.run(d)
	print "Document Title: %s \t Number of Annotated Entities %s \t Number of Annotated Triples %s" % (d.title, len(d.entities), len(d.triples))

 #   except Exception as e:

  #      print "error Processing document %s" % d.title
<|endoftext|>


<commit_before>import tornado.gen

from . import event

PHASE_CONFIGURATION = event.Event()
PHASE_SETUP = event.Event()
PHASE_START = event.Event()
PHASE_POST_START = event.Event()

@tornado.gen.coroutine
def start():
    yield PHASE_CONFIGURATION()
    yield PHASE_SETUP()
    yield PHASE_START()
    yield PHASE_POST_START()<commit_msg>no more code in __init__.py<commit_after><|endoftext|>


<commit_before># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
# OpenFlow 1.2 upgrade, modified for peering exchanges by Sam Russell 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.topology import event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.controller.handler import set_ev_handler
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser, ether
from ryu.lib.mac import haddr_to_str
from ryu.lib import ip, packet

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SPE(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SPE, self).__init__(*args, **kwargs)
        self.dps = {}
        self.mac_to_port = {}

    def add_flow(self, datapath, table_id, match, instructions, priority=0x8000, buffer_id=ofproto_v1_2.OFP_NO_BUFFER):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        #datapath, cookie, cookie_mask, table_id, command,
        #         idle_timeout, hard_timeout, priority, buffer_id, out_port,
        #         out_group, flags, match, instructions)
        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, cookie=0, cookie_mask=0, table_id=table_id,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000, buffer_id=buffer_id,
            out_port=ofproto_v1_2.OFPP_ANY, out_group=ofproto_v1_2.OFPG_ANY,
            flags=ofproto.OFPFF_SEND_FLOW_REM, match=match, instructions=instructions)
        datapath.send_msg(mod)

    def add_arp_reply_catcher(self, datapath, ip = None, port = None, accept = False, table_id=0):
        match = datapath.ofproto_parser.OFPMatch()
        match.set_dl_type(ether.ETH_TYPE_ARP)
        match.set_arp_opcode(2)
        if port:
            match.set_in_port(port)
        if ip:
            match.set_arp_spa(ip)
        ofproto = datapath.ofproto
        if accept:
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(2)]
        else:
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(1)]
        self.add_flow(datapath, table_id, match, instructions, priority=0x8000)

    def init_flows(self, datapath):
        ofproto = datapath.ofproto
        # send arp replies to controller always
        self.add_arp_reply_catcher(datapath)
        self.add_arp_reply_catcher(datapath, ip=ip.ipv4_to_bin('10.1.1.1'), port=1, accept=True, table_id=1)
        self.add_arp_reply_catcher(datapath, ip=ip.ipv4_to_bin('10.1.1.2'), port=2, accept=True, table_id=1)
        # make a flow to flood ARP packets
        match = datapath.ofproto_parser.OFPMatch()
        match.set_dl_type(ether.ETH_TYPE_ARP)
        match.set_arp_opcode(1)
        actions = [datapath.ofproto_parser.OFPActionOutput(ofproto.OFPP_FLOOD, 1500)]
        instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
        self.add_flow(datapath, 0, match, instructions)

    @set_ev_cls(ofp_event.EventOFPStateChange, [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def switch_enter_handler(self, ev):
        dp = ev.datapath
        if ev.state == MAIN_DISPATCHER:
            self.logger.info("Switch entered: %s", dp.id)
            self.init_flows(dp)

        elif ev.state == DEAD_DISPATCHER:
            if dp.id is None:
                return
            self.logger.info("Switch left: %s", dp.id)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        ethtype = 0
        #iterate through fields - parser should handle this
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        # we should build a dictionary of MTXXXX class names to speed this up
        for o in match.fields:
            self.logger.info("Field %s", str(o))
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # parse packet

        pkt = packet.Packet(data)
        self.logger.info("Packet ethertype: %s", pkt.ethertype)

        ## is it an arp reply?
        #for o in match.fields:
        #    if isinstance(o, ofproto_v1_2_parser.MTArpOp):
        #        if o.value == 2:
        #            # drop arp reply if it's gotten to the controller
        #            return
        #
        ## get ethertype
        #for o in match.fields:
        #    if isinstance(o, ofproto_v1_2_parser.MTEthType):
        #        ethtype = o.value
        #        break

        # if ARP (request) then flood and don't make a flow
        if ethtype == ether.ETH_TYPE_ARP:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

        # do we know the mac?
        if src not in self.mac_to_port[dpid]:
            # learn the mac address to avoid FLOOD next time.
            self.mac_to_port[dpid][src] = in_port
            # set a flow to table 0 to allow packets through to table 1
            match = datapath.ofproto_parser.OFPMatch()
            match.set_in_port(in_port)
            match.set_dl_src(src)
            match.set_dl_type(ethtype)
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(2)]
            self.add_flow(datapath, 0, match, instructions)

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
            match = datapath.ofproto_parser.OFPMatch()
            match.set_dl_dst(dst)
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
            self.add_flow(datapath, 2, match, instructions, buffer_id=msg.buffer_id)
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<commit_msg>import the right packet<commit_after># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
# OpenFlow 1.2 upgrade, modified for peering exchanges by Sam Russell 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.topology import event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.controller.handler import set_ev_handler
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser, ether
from ryu.lib.mac import haddr_to_str
from ryu.lib import ip
from ryu.lib.packet import packet

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SPE(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SPE, self).__init__(*args, **kwargs)
        self.dps = {}
        self.mac_to_port = {}

    def add_flow(self, datapath, table_id, match, instructions, priority=0x8000, buffer_id=ofproto_v1_2.OFP_NO_BUFFER):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        #datapath, cookie, cookie_mask, table_id, command,
        #         idle_timeout, hard_timeout, priority, buffer_id, out_port,
        #         out_group, flags, match, instructions)
        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, cookie=0, cookie_mask=0, table_id=table_id,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000, buffer_id=buffer_id,
            out_port=ofproto_v1_2.OFPP_ANY, out_group=ofproto_v1_2.OFPG_ANY,
            flags=ofproto.OFPFF_SEND_FLOW_REM, match=match, instructions=instructions)
        datapath.send_msg(mod)

    def add_arp_reply_catcher(self, datapath, ip = None, port = None, accept = False, table_id=0):
        match = datapath.ofproto_parser.OFPMatch()
        match.set_dl_type(ether.ETH_TYPE_ARP)
        match.set_arp_opcode(2)
        if port:
            match.set_in_port(port)
        if ip:
            match.set_arp_spa(ip)
        ofproto = datapath.ofproto
        if accept:
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(2)]
        else:
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(1)]
        self.add_flow(datapath, table_id, match, instructions, priority=0x8000)

    def init_flows(self, datapath):
        ofproto = datapath.ofproto
        # send arp replies to controller always
        self.add_arp_reply_catcher(datapath)
        self.add_arp_reply_catcher(datapath, ip=ip.ipv4_to_bin('10.1.1.1'), port=1, accept=True, table_id=1)
        self.add_arp_reply_catcher(datapath, ip=ip.ipv4_to_bin('10.1.1.2'), port=2, accept=True, table_id=1)
        # make a flow to flood ARP packets
        match = datapath.ofproto_parser.OFPMatch()
        match.set_dl_type(ether.ETH_TYPE_ARP)
        match.set_arp_opcode(1)
        actions = [datapath.ofproto_parser.OFPActionOutput(ofproto.OFPP_FLOOD, 1500)]
        instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
        self.add_flow(datapath, 0, match, instructions)

    @set_ev_cls(ofp_event.EventOFPStateChange, [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def switch_enter_handler(self, ev):
        dp = ev.datapath
        if ev.state == MAIN_DISPATCHER:
            self.logger.info("Switch entered: %s", dp.id)
            self.init_flows(dp)

        elif ev.state == DEAD_DISPATCHER:
            if dp.id is None:
                return
            self.logger.info("Switch left: %s", dp.id)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        ethtype = 0
        #iterate through fields - parser should handle this
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        # we should build a dictionary of MTXXXX class names to speed this up
        for o in match.fields:
            self.logger.info("Field %s", str(o))
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # parse packet

        pkt = packet.Packet(data)
        self.logger.info("Packet ethertype: %s", pkt.ethertype)

        ## is it an arp reply?
        #for o in match.fields:
        #    if isinstance(o, ofproto_v1_2_parser.MTArpOp):
        #        if o.value == 2:
        #            # drop arp reply if it's gotten to the controller
        #            return
        #
        ## get ethertype
        #for o in match.fields:
        #    if isinstance(o, ofproto_v1_2_parser.MTEthType):
        #        ethtype = o.value
        #        break

        # if ARP (request) then flood and don't make a flow
        if ethtype == ether.ETH_TYPE_ARP:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

        # do we know the mac?
        if src not in self.mac_to_port[dpid]:
            # learn the mac address to avoid FLOOD next time.
            self.mac_to_port[dpid][src] = in_port
            # set a flow to table 0 to allow packets through to table 1
            match = datapath.ofproto_parser.OFPMatch()
            match.set_in_port(in_port)
            match.set_dl_src(src)
            match.set_dl_type(ethtype)
            instructions = [datapath.ofproto_parser.OFPInstructionGotoTable(2)]
            self.add_flow(datapath, 0, match, instructions)

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
            match = datapath.ofproto_parser.OFPMatch()
            match.set_dl_dst(dst)
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]
            self.add_flow(datapath, 2, match, instructions, buffer_id=msg.buffer_id)
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD
            actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
            out = datapath.ofproto_parser.OFPPacketOut(
                datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
                actions=actions)
            datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Find sales prices of various types of properties sold in a given
period of time and prepare for plotting on a map with R.
"""

# It is possible that the module 'vincent' could do the plotting

import requests
import bs4
import datetime
import math

currentYear = datetime.datetime.now().year
propertyTypes = ['villa', 'ejerlejlighed']

def numPages(firstYear, lastYear, propType):
    """ Find the number of pages we have to iterate through to get all the
    results we want.
    Each page contains 40 results.
    """
    # Get the page and check the result is good
    resPropNo = requests.get('http://www.boliga.dk/salg/resultater?so=1&type='
                             + propType +
                             '&fraPostnr=1000&tilPostnr=9990&minsaledate=' +
                             firstYear + '&maxsaledate=' + lastYear)
    resPropNo.raise_for_status()

    # Parse the result
    propNoSoup = bs4.BeautifulSoup(resPropNo.text, 'html.parser')

    # Extract the total number of results from the text above the results table
    propNoArr = propNoSoup.select('td[class="text-center"] > label'
                                  )[0].getText().split()

    for i in range(len(propNoArr)):
        try:
            # Is the element a number?
            int(propNoArr[i])
        except ValueError:
            # Not a number, next element
            continue
        # The only element that is a number is the total number of results
        propNo = propNoArr[i]

    # Return value is the number of pages with 40 results per page
    pageNo = math.ceil(propNo, 40)

    return pageNo

while True:
    try:
        # We should probably default to the current year
        startYear = int(input("Starting year: "))
    except ValueError:
        # User did not enter a valid year
        print("Please enter a year between 1992 and " + str(currentYear) + ".")
        # Back to start
        continue
    if startYear < 1992 or startYear > currentYear:
        # There are no data before 1992 or for the future
        print("Please enter a year between 1992 and " + str(currentYear) + ".")
        # Back to start
        continue
    else:
        # Success
        break

if startYear == currentYear:
    endYear = 'today'
else:
    while True:
        endYear = input("End year (or leave blank for today): ")
        if endYear == '' or endYear == 'today':
            # No input, end date is today, also silently accept 'today'
            endYear = 'today'
            break
        try:
            # Check if user input is an integer
            int(endYear)
        except ValueError:
            # Not an integer, back to start
            print("Please enter a year between " + str(startYear+1) + " and "
                  + currentYear + ".")
            continue
        if int(endYear) <= startYear:
            print("End year must be greater than starting year (" +
                  str(startYear) + ").")
            continue
        elif int(endYear) > currentYear:
            # Assume user wants all data until today
            print("Future date given, assuming today.")
            endYear = 'today'
            break
        else:
            # All is good
            break

# Preliminary output to validate the above
print("This program should fetch prices from " + str(startYear) + " until " +
      str(endYear) + ".")
<commit_msg>Style cleanup.<commit_after># -*- coding: utf-8 -*-
"""Find sales prices of various types of properties sold in a given
period of time and prepare for plotting on a map with R.
"""

# It is possible that the module 'vincent' could do the plotting

import requests
import bs4
import datetime
import math

currentYear = datetime.datetime.now().year
propertyTypes = ['villa', 'ejerlejlighed']

def numPages(firstYear, lastYear, propType):
    """ Find the number of pages we have to iterate through to get all the
    results we want. Each page contains 40 results; this should probably be
    verified by looking at the text.
    """
    # Get the page and check the result is good
    resPropNo = requests.get('http://www.boliga.dk/salg/resultater?so=1&type='
                             + propType +
                             '&fraPostnr=1000&tilPostnr=9990&minsaledate=' +
                             firstYear + '&maxsaledate=' + lastYear)
    resPropNo.raise_for_status()

    # Parse the result
    propNoSoup = bs4.BeautifulSoup(resPropNo.text, 'html.parser')

    # Extract the total number of results from the text above the results table
    propNoArr = propNoSoup.select('td[class="text-center"] > label'
                                  )[0].getText().split()

    for i in range(len(propNoArr)):
        try:
            # Is the element a number?
            int(propNoArr[i])
        except ValueError:
            # Not a number, next element
            continue
        # The only element that is a number is the total number of results
        propNo = propNoArr[i]

    # Return value is the number of pages with 40 results per page
    pageNo = math.ceil(propNo, 40)

    return pageNo

def getPrices(firstYear, lastYear, propType):
    """ Get the prices of the given property type for the given years. Also
    extract the addresses, we will use these to find the geographic
    coordinates of the properties later.
    """

while True:
    try:
        # We should probably default to the current year
        startYear = int(input("Starting year: "))
    except ValueError:
        # User did not enter a valid year
        print("Please enter a year between 1992 and " + str(currentYear) + ".")
        # Back to start
        continue
    if startYear < 1992 or startYear > currentYear:
        # There are no data before 1992 or for the future
        print("Please enter a year between 1992 and " + str(currentYear) + ".")
        # Back to start
        continue
    else:
        # Success
        break

if startYear == currentYear:
    endYear = 'today'
else:
    while True:
        endYear = input("End year (or leave blank for today): ")
        if endYear == '' or endYear == 'today':
            # No input, end date is today, also silently accept 'today'
            endYear = 'today'
            break
        try:
            # Check if user input is an integer
            int(endYear)
        except ValueError:
            # Not an integer, back to start
            print("Please enter a year between " + str(startYear+1) + " and "
                  + currentYear + ".")
            continue
        if int(endYear) <= startYear:
            print("End year must be greater than starting year (" +
                  str(startYear) + ").")
            continue
        elif int(endYear) > currentYear:
            # Assume user wants all data until today
            print("Future date given, assuming today.")
            endYear = 'today'
            break
        else:
            # All is good
            break

# Preliminary output to validate the above
print("This program should fetch prices from " + str(startYear) + " until " +
      str(endYear) + ".")
<|endoftext|>


<commit_before>'''
All salt configuration loading and defaults should be in this module
'''
# Import python modules
import os
import socket
import subprocess
import logging
# Import third party libs
import yaml
# Import salt libs
import salt.crypt
import salt.loader

def minion_config(path):
    '''
    Reads in the minion configuration file and sets up special options
    '''
    opts = {'master': 'mcp',
            'master_port': '4506',
            'pki_dir': '/etc/salt/pki',
            'id': socket.getfqdn(),
            'cachedir': '/var/cache/salt',
            'conf_file': path,
            'disable_modules': [],
            'disable_returners': [],
            'module_dirs': [],
            'returner_dirs': [],
            'open_mode': False,
            'multiprocessing': False,
            'log_file': '/var/log/salt/minion',
            'log_level': 'WARNING',
            'out_level': 'ERROR',
            }

    if os.path.isfile(path):
        try:
            opts.update(yaml.load(open(path, 'r')))
        except Exception:
            pass

    opts['master_uri'] = 'tcp://' + opts['master'] + ':'\
                       + str(opts['master_port'])

    # Enableing open mode requires that the value be set to True, and nothing
    # else!
    if opts['open_mode']:
        if opts['open_mode'] == True:
            opts['open_mode'] = True
        else:
            opts['open_mode'] = False

    opts['logger'] = master_logger(opts['log_file'],
                                   opts['log_level'],
                                   opts['out_level'])

    opts['grains'] = salt.loader.grains()

    return opts

def master_config(path):
    '''
    Reads in the master configuration file and sets up default options
    '''
    opts = {'interface': '0.0.0.0',
            'publish_port': '4505',
            'publish_pull_port': '45055',
            'worker_threads': 5,
            'worker_start_port': '45056',
            'ret_port': '4506',
            'keep_jobs': 24,
            'pki_dir': '/etc/salt/pki',
            'cachedir': '/var/cache/salt',
            'conf_file': path,
            'open_mode': False,
            'auto_accept': False,
            'log_file': '/var/log/salt/master',
            'log_level': 'WARNING',
            'out_level': 'ERROR',
            'cluster_masters': [],
            'cluster_mode': 'paranoid',
            }

    if os.path.isfile(path):
        try:
            opts.update(yaml.load(open(path, 'r')))
        except Exception:
            pass

    opts['aes'] = salt.crypt.Crypticle.generate_key_string()

    # Enableing open mode requires that the value be set to True, and nothing
    # else!
    if opts['open_mode']:
        if opts['open_mode'] == True:
            opts['open_mode'] = True
        else:
            opts['open_mode'] = False
    if opts['auto_accept']:
        if opts['auto_accept'] == True:
            opts['auto_accept'] = True
        else:
            opts['auto_accept'] = False

    opts['logger'] = master_logger(opts['log_file'],
                                   opts['log_level'],
                                   opts['out_level'])

    return opts

def master_logger(log_file, log_level, console_level):
    '''
    Returns a logger fo use with a salt master
    '''
    if not os.path.isdir(os.path.dirname(log_file)):
        os.makedirs(os.path.dirname(log_file))
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    fh_ = logging.FileHandler(log_file)
    fh_.setLevel(getattr(logging, log_level))

    ch_ = logging.StreamHandler()
    ch_.setLevel(getattr(logging, console_level))

    fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    formatter = logging.Formatter(fmt)
    ch_.setFormatter(formatter)
    fh_.setFormatter(formatter)
    logger.addHandler(ch_)
    logger.addHandler(fh_)

    return logger

def minion_logger(log_file, log_level, console_level):
    '''
    Returns a logger fo use with a salt minion
    '''
    if not os.path.isdir(os.path.dirname(log_file)):
        os.makedirs(os.path.dirname(log_file))
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    fh_ = logging.FileHandler(log_file)
    fh_.setLevel(getattr(logging, log_level))

    ch_ = logging.StreamHandler()
    ch_.setLevel(getattr(logging, console_level))

    fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    formatter = logging.Formatter(fmt)
    ch_.setFormatter(formatter)
    fh_.setFormatter(formatter)
    logger.addHandler(ch_)
    logger.addHandler(fh_)

    return logger

<commit_msg>Set default values for new configuration params<commit_after>'''
All salt configuration loading and defaults should be in this module
'''
# Import python modules
import os
import socket
import subprocess
import logging
# Import third party libs
import yaml
# Import salt libs
import salt.crypt
import salt.loader

def minion_config(path):
    '''
    Reads in the minion configuration file and sets up special options
    '''
    opts = {'master': 'mcp',
            'master_port': '4506',
            'pki_dir': '/etc/salt/pki',
            'id': socket.getfqdn(),
            'cachedir': '/var/cache/salt',
            'conf_file': path,
            'renderer': 'yaml_jinja',
            'disable_modules': [],
            'disable_returners': [],
            'module_dirs': [],
            'returner_dirs': [],
            'states_dirs': [],
            'render_dirs': [],
            'open_mode': False,
            'multiprocessing': False,
            'log_file': '/var/log/salt/minion',
            'log_level': 'WARNING',
            'out_level': 'ERROR',
            }

    if os.path.isfile(path):
        try:
            opts.update(yaml.load(open(path, 'r')))
        except Exception:
            pass

    opts['master_uri'] = 'tcp://' + opts['master'] + ':'\
                       + str(opts['master_port'])

    # Enableing open mode requires that the value be set to True, and nothing
    # else!
    if opts['open_mode']:
        if opts['open_mode'] == True:
            opts['open_mode'] = True
        else:
            opts['open_mode'] = False

    opts['logger'] = master_logger(opts['log_file'],
                                   opts['log_level'],
                                   opts['out_level'])

    opts['grains'] = salt.loader.grains()

    return opts

def master_config(path):
    '''
    Reads in the master configuration file and sets up default options
    '''
    opts = {'interface': '0.0.0.0',
            'publish_port': '4505',
            'publish_pull_port': '45055',
            'worker_threads': 5,
            'worker_start_port': '45056',
            'ret_port': '4506',
            'keep_jobs': 24,
            'pki_dir': '/etc/salt/pki',
            'cachedir': '/var/cache/salt',
            'conf_file': path,
            'open_mode': False,
            'auto_accept': False,
            'log_file': '/var/log/salt/master',
            'log_level': 'WARNING',
            'out_level': 'ERROR',
            'cluster_masters': [],
            'cluster_mode': 'paranoid',
            }

    if os.path.isfile(path):
        try:
            opts.update(yaml.load(open(path, 'r')))
        except Exception:
            pass

    opts['aes'] = salt.crypt.Crypticle.generate_key_string()

    # Enableing open mode requires that the value be set to True, and nothing
    # else!
    if opts['open_mode']:
        if opts['open_mode'] == True:
            opts['open_mode'] = True
        else:
            opts['open_mode'] = False
    if opts['auto_accept']:
        if opts['auto_accept'] == True:
            opts['auto_accept'] = True
        else:
            opts['auto_accept'] = False

    opts['logger'] = master_logger(opts['log_file'],
                                   opts['log_level'],
                                   opts['out_level'])

    return opts

def master_logger(log_file, log_level, console_level):
    '''
    Returns a logger fo use with a salt master
    '''
    if not os.path.isdir(os.path.dirname(log_file)):
        os.makedirs(os.path.dirname(log_file))
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    fh_ = logging.FileHandler(log_file)
    fh_.setLevel(getattr(logging, log_level))

    ch_ = logging.StreamHandler()
    ch_.setLevel(getattr(logging, console_level))

    fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    formatter = logging.Formatter(fmt)
    ch_.setFormatter(formatter)
    fh_.setFormatter(formatter)
    logger.addHandler(ch_)
    logger.addHandler(fh_)

    return logger

def minion_logger(log_file, log_level, console_level):
    '''
    Returns a logger fo use with a salt minion
    '''
    if not os.path.isdir(os.path.dirname(log_file)):
        os.makedirs(os.path.dirname(log_file))
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    fh_ = logging.FileHandler(log_file)
    fh_.setLevel(getattr(logging, log_level))

    ch_ = logging.StreamHandler()
    ch_.setLevel(getattr(logging, console_level))

    fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    formatter = logging.Formatter(fmt)
    ch_.setFormatter(formatter)
    fh_.setFormatter(formatter)
    logger.addHandler(ch_)
    logger.addHandler(fh_)

    return logger

<|endoftext|>


<commit_before>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import shutil
import threading
import multiprocessing
import time
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptogrogphy modules
from M2Crypto import RSA

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        if self.opts['keep_jobs'] == 0:
            return
        diff = self.opts['keep_jobs'] * 60 * 60
        keep = int(time.time()) - diff
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        for jid in os.listdir(jid_root):
            if int(jid.split('.')[0]) < keep:
                shutil.rmtree(os.path.join(jid_root, jid))

    def start(self):
        '''
        Turn on the master server components
        '''
        self.opts['logger'].info('Starting the Salt Master')
        reqserv = ReqServer(self.opts)
        reqserv.run()
        while True:
            self._clear_old_jobs()
            time.sleep(60)

class Publisher(multiprocessing.Process):
    '''
    The publihing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        multiprocessing.Process.__init__(self)
        self.opts = opts

    def run(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUB)
        rep_sock = context.socket(zmq.REP)
        pub_uri = 'tcp://' + self.opts['interface'] + ':'\
               + self.opts['publish_port']
        rep_uri = 'tcp://127.0.0.1:' + self.opts['publish_rep_port']
        self.opts['logger'].info('Starting the Salt Publisher on ' + pub_uri)
        pub_sock.bind(pub_uri)
        rep_sock.bind(rep_uri)

        while True:
            package = rep_sock.recv()
            rep_sock.send('')
            self.opts['logger'].info('Publishing command')
            pub_sock.send(package)

class ReqServer():
    '''
    Starts up the master request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://' + self.opts['interface'] + ':'\
                   + self.opts['ret_port']
        self.clients = self.context.socket(zmq.XREP)
        self.workers = self.context.socket(zmq.XREQ)
        self.w_uri = 'inproc://workers'
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        self.opts['logger'].info('Preparing the root key for local'\
                + ' comunication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __worker(self):
        '''
        Starts up a worker thread
        '''
        in_socket = self.context.socket(zmq.REP)
        in_socket.connect(self.w_uri)
        m_worker = MWorker(self.opts,
                self.master_key,
                self.key,
                self.crypticle)
        work_port = m_worker.port
        m_worker.start()

        out_socket = self.context.socket(zmq.REQ)
        out_socket.connect('tcp://127.0.0.1:' + str(work_port))

        while True:
            package = in_socket.recv()
            out_socket.send(package)
            ret = out_socket.recv()
            in_socket.send(ret)

    def __bind(self):
        '''
        Binds the reply server
        '''
        self.opts['logger'].info('Setting up the master communication server')
        self.clients.bind(self.uri)

        self.workers.bind(self.w_uri)

        for ind in range(int(self.opts['worker_threads'])):
            threading.Thread(target=self.__worker).start()

        zmq.device(zmq.QUEUE, self.clients, self.workers)

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

class MWorker(multiprocessing.Process):
    '''
    The worker multiprocess instance to manage the backend operations for the
    salt master.
    '''
    def __init__(self, opts, mkey, key, crypticle):
        multiprocessing.Process.__init__(self)
        self.opts = opts
        self.master_key = mkey
        self.key = key
        self.crypticle = crypticle
        self.port = str(int(self.name.split('-')[1]) - 2 + int(self.opts['worker_start_port']))

    def __bind(self):
        '''
        Bind to the local port
        '''
        context = zmq.Context(1)
        socket = context.socket(zmq.REP)
        socket.bind('tcp://127.0.0.1:' + self.port)

        while True:
            package = socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = datetime.datetime.strftime(datetime.datetime.now(),
                '%Y%m%d%H%M%S%f')
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        self.opts['logger'].info('Clear payload recieved with command '\
                + load['cmd'])
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        self.opts['logger'].info('Pubkey payload recieved with command '\
                + load['cmd'])

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        self.opts['logger'].info('AES payload recieved with command '\
                + data['cmd'])
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are recieving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        self.opts['logger'].info('Authentication request from '\
                + load['id'])
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            self.opts['logger'].info('New public key placed in pending for '\
                    + load['id'])
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys in'\
                        + ' pending did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                self.opts['logger'].info('Authentication failed from host '\
                        + load['id'] + ', the key is in pending and'\
                        + ' needs to be accepted with saltkey -a '\
                        + load['id'])
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        self.opts['logger'].info('Authentication accepted from '\
                + load['id'])
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        self.opts['logger'].info('Got return from ' + load['id']\
                + ' for job ' + load['jid'])
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            self.opts['logger'].error('An inconsistency occured, a job was'\
                    + ' recieved with a job id that is not present on the'\
                    + ' master: ' + load['jid'])
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluser data out
        '''
        self.opts['logger'].debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        self.opts['logger'].debug('Cluster distributed: ' + str(ret))

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.REQ)
        pub_sock.connect('tcp://127.0.0.1:' + self.opts['publish_rep_port'])
        pub_sock.send(salt.payload.package(payload))
        pub_sock.recv()
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start a Master Worker
        '''
        self.__bind()
<commit_msg>Add import datetime<commit_after>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import shutil
import threading
import multiprocessing
import time
import datetime
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptogrogphy modules
from M2Crypto import RSA

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        if self.opts['keep_jobs'] == 0:
            return
        diff = self.opts['keep_jobs'] * 60 * 60
        keep = int(time.time()) - diff
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        for jid in os.listdir(jid_root):
            if int(jid.split('.')[0]) < keep:
                shutil.rmtree(os.path.join(jid_root, jid))

    def start(self):
        '''
        Turn on the master server components
        '''
        self.opts['logger'].info('Starting the Salt Master')
        reqserv = ReqServer(self.opts)
        reqserv.run()
        while True:
            self._clear_old_jobs()
            time.sleep(60)

class Publisher(multiprocessing.Process):
    '''
    The publihing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        multiprocessing.Process.__init__(self)
        self.opts = opts

    def run(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUB)
        rep_sock = context.socket(zmq.REP)
        pub_uri = 'tcp://' + self.opts['interface'] + ':'\
               + self.opts['publish_port']
        rep_uri = 'tcp://127.0.0.1:' + self.opts['publish_rep_port']
        self.opts['logger'].info('Starting the Salt Publisher on ' + pub_uri)
        pub_sock.bind(pub_uri)
        rep_sock.bind(rep_uri)

        while True:
            package = rep_sock.recv()
            rep_sock.send('')
            self.opts['logger'].info('Publishing command')
            pub_sock.send(package)

class ReqServer():
    '''
    Starts up the master request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://' + self.opts['interface'] + ':'\
                   + self.opts['ret_port']
        self.clients = self.context.socket(zmq.XREP)
        self.workers = self.context.socket(zmq.XREQ)
        self.w_uri = 'inproc://workers'
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        self.opts['logger'].info('Preparing the root key for local'\
                + ' comunication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __worker(self):
        '''
        Starts up a worker thread
        '''
        in_socket = self.context.socket(zmq.REP)
        in_socket.connect(self.w_uri)
        m_worker = MWorker(self.opts,
                self.master_key,
                self.key,
                self.crypticle)
        work_port = m_worker.port
        m_worker.start()

        out_socket = self.context.socket(zmq.REQ)
        out_socket.connect('tcp://127.0.0.1:' + str(work_port))

        while True:
            package = in_socket.recv()
            out_socket.send(package)
            ret = out_socket.recv()
            in_socket.send(ret)

    def __bind(self):
        '''
        Binds the reply server
        '''
        self.opts['logger'].info('Setting up the master communication server')
        self.clients.bind(self.uri)

        self.workers.bind(self.w_uri)

        for ind in range(int(self.opts['worker_threads'])):
            threading.Thread(target=self.__worker).start()

        zmq.device(zmq.QUEUE, self.clients, self.workers)

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

class MWorker(multiprocessing.Process):
    '''
    The worker multiprocess instance to manage the backend operations for the
    salt master.
    '''
    def __init__(self, opts, mkey, key, crypticle):
        multiprocessing.Process.__init__(self)
        self.opts = opts
        self.master_key = mkey
        self.key = key
        self.crypticle = crypticle
        self.port = str(int(self.name.split('-')[1]) - 2 + int(self.opts['worker_start_port']))

    def __bind(self):
        '''
        Bind to the local port
        '''
        context = zmq.Context(1)
        socket = context.socket(zmq.REP)
        socket.bind('tcp://127.0.0.1:' + self.port)

        while True:
            package = socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = datetime.datetime.strftime(datetime.datetime.now(),
                '%Y%m%d%H%M%S%f')
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        self.opts['logger'].info('Clear payload recieved with command '\
                + load['cmd'])
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        self.opts['logger'].info('Pubkey payload recieved with command '\
                + load['cmd'])

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        self.opts['logger'].info('AES payload recieved with command '\
                + data['cmd'])
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are recieving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        self.opts['logger'].info('Authentication request from '\
                + load['id'])
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            self.opts['logger'].info('New public key placed in pending for '\
                    + load['id'])
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys in'\
                        + ' pending did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                self.opts['logger'].info('Authentication failed from host '\
                        + load['id'] + ', the key is in pending and'\
                        + ' needs to be accepted with saltkey -a '\
                        + load['id'])
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        self.opts['logger'].info('Authentication accepted from '\
                + load['id'])
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        self.opts['logger'].info('Got return from ' + load['id']\
                + ' for job ' + load['jid'])
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            self.opts['logger'].error('An inconsistency occured, a job was'\
                    + ' recieved with a job id that is not present on the'\
                    + ' master: ' + load['jid'])
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluser data out
        '''
        self.opts['logger'].debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        self.opts['logger'].debug('Cluster distributed: ' + str(ret))

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.REQ)
        pub_sock.connect('tcp://127.0.0.1:' + self.opts['publish_rep_port'])
        pub_sock.send(salt.payload.package(payload))
        pub_sock.recv()
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start a Master Worker
        '''
        self.__bind()
<|endoftext|>