<commit_before>import os
import sys
import unittest

import tests.utils
from tests.utils import cat

class GenerateOutputTest(unittest.TestCase):
    def snippet_call_generate_output(self, args, input_files, expected_values, disallowed_files=None):
        with tests.utils.sandbox(input_files) as tempdir:
            tests.utils.run(['generate-output'] + args, check=True)
            for expect in expected_values:
                with open(expect['path']) as f:
                    self.assertEqual(''.join(f.readlines()), expect['data'])
            if disallowed_files is not None:
                for file in disallowed_files:
                    self.assertFalse(os.path.exists(file))

    def test_call_generate_output_simple(self):
        self.snippet_call_generate_output(
            args=['-c', cat()],
            input_files=[
                {
                    'path': 'test/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-2.in',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'test/sample-1.out',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-2.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_select(self):
        self.snippet_call_generate_output(args=['-c', cat(), 'test/sample-1.in', 'test/sample-2.in'], input_files=[
            {
                'path': 'test/sample-1.in',
                'data': 'foo\n'
            },
            {
                'path': 'test/sample-2.in',
                'data': 'bar\n'
            },
            {
                'path': 'test/sample-3.in',
                'data': 'baz\n'
            },
        ], expected_values=[
            {
                'path': 'test/sample-1.out',
                'data': 'foo\n'
            },
            {
                'path': 'test/sample-2.out',
                'data': 'bar\n'
            },
        ], disallowed_files=['test/sample-3.out'])

    def test_call_generate_output_already_exists(self):
        # Since sample-1.out already exists, sample-1.out will not be updated.
        self.snippet_call_generate_output(
            args=['-c', cat()],
            input_files=[
                {
                    'path': 'test/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-1.out',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'test/sample-1.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_dir(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'p/o/../../p/o/y/o'],
            input_files=[
                {
                    'path': 'p/o/y/o/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'p/o/y/o/sample-2.in',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'p/o/y/o/sample-1.out',
                    'data': 'foo\n'
                },
                {
                    'path': 'p/o/y/o/sample-2.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_format(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'yuki/coder', '-f', 'test_%e/%s'],
            input_files=[
                {
                    'path': 'yuki/coder/test_in/sample-1.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'yuki/coder/test_in/sample-2.txt',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'yuki/coder/test_out/sample-1.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'yuki/coder/test_out/sample-2.txt',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_format_select(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'yuki/coder', '-f', 'test_%e/%s', 'yuki/coder/test_in/sample-2.txt', 'yuki/coder/test_in/sample-3.txt'],
            input_files=[
                {
                    'path': 'yuki/coder/test_in/sample-2.txt',
                    'data': 'bar\n'
                },
                {
                    'path': 'yuki/coder/test_in/sample-3.txt',
                    'data': 'baz\n'
                },
            ],
            expected_values=[
                {
                    'path': 'yuki/coder/test_out/sample-2.txt',
                    'data': 'bar\n'
                },
                {
                    'path': 'yuki/coder/test_out/sample-3.txt',
                    'data': 'baz\n'
                },
            ],
        )

    def test_call_generate_output_format_hack(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'a/b', '-f', 'c/test_%e/d/%s/e.case.txt'],
            input_files=[
                {
                    'path': 'a/b/c/test_in/d/sample.case.1/e.case.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'a/b/c/test_in/d/sample.case.2/e.case.txt',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'a/b/c/test_out/d/sample.case.1/e.case.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'a/b/c/test_out/d/sample.case.2/e.case.txt',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_in_parallel(self):
        if os.name == 'nt':
            TOTAL = 50
            PARALLEL = 8
        else:
            TOTAL = 1000
            PARALLEL = 256
        input_files = []
        expected_values = []
        for i in range(TOTAL):
            name = 'sample-%03d' % i
            input_files += [{
                'path': 'test/{}.in'.format(name),
                'data': str(i),
            }]
            if i > TOTAL * 0.95:
                input_files += [{
                    'path': 'test/{}.out'.format(name),
                    'data': str(i),
                }]
            expected_values += [{
                'path': 'test/{}.in'.format(name),
                'data': str(i),
            }]
            expected_values += [{
                'path': 'test/{}.out'.format(name),
                'data': str(i),
            }]
        self.snippet_call_generate_output(
            args=['--jobs', str(PARALLEL), '-c', tests.utils.python_c("import sys, time; time.sleep(1); sys.stdout.write(sys.stdin.read())")],
            input_files=input_files,
            expected_values=expected_values,
        )
<commit_msg>#468: remove unnecessary branch.<commit_after>import os
import sys
import unittest

import tests.utils
from tests.utils import cat

class GenerateOutputTest(unittest.TestCase):
    def snippet_call_generate_output(self, args, input_files, expected_values, disallowed_files=None):
        with tests.utils.sandbox(input_files) as tempdir:
            tests.utils.run(['generate-output'] + args, check=True)
            for expect in expected_values:
                with open(expect['path']) as f:
                    self.assertEqual(''.join(f.readlines()), expect['data'])
            if disallowed_files is not None:
                for file in disallowed_files:
                    self.assertFalse(os.path.exists(file))

    def test_call_generate_output_simple(self):
        self.snippet_call_generate_output(
            args=['-c', cat()],
            input_files=[
                {
                    'path': 'test/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-2.in',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'test/sample-1.out',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-2.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_select(self):
        self.snippet_call_generate_output(args=['-c', cat(), 'test/sample-1.in', 'test/sample-2.in'], input_files=[
            {
                'path': 'test/sample-1.in',
                'data': 'foo\n'
            },
            {
                'path': 'test/sample-2.in',
                'data': 'bar\n'
            },
            {
                'path': 'test/sample-3.in',
                'data': 'baz\n'
            },
        ], expected_values=[
            {
                'path': 'test/sample-1.out',
                'data': 'foo\n'
            },
            {
                'path': 'test/sample-2.out',
                'data': 'bar\n'
            },
        ], disallowed_files=['test/sample-3.out'])

    def test_call_generate_output_already_exists(self):
        # Since sample-1.out already exists, sample-1.out will not be updated.
        self.snippet_call_generate_output(
            args=['-c', cat()],
            input_files=[
                {
                    'path': 'test/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'test/sample-1.out',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'test/sample-1.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_dir(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'p/o/../../p/o/y/o'],
            input_files=[
                {
                    'path': 'p/o/y/o/sample-1.in',
                    'data': 'foo\n'
                },
                {
                    'path': 'p/o/y/o/sample-2.in',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'p/o/y/o/sample-1.out',
                    'data': 'foo\n'
                },
                {
                    'path': 'p/o/y/o/sample-2.out',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_format(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'yuki/coder', '-f', 'test_%e/%s'],
            input_files=[
                {
                    'path': 'yuki/coder/test_in/sample-1.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'yuki/coder/test_in/sample-2.txt',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'yuki/coder/test_out/sample-1.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'yuki/coder/test_out/sample-2.txt',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_format_select(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'yuki/coder', '-f', 'test_%e/%s', 'yuki/coder/test_in/sample-2.txt', 'yuki/coder/test_in/sample-3.txt'],
            input_files=[
                {
                    'path': 'yuki/coder/test_in/sample-2.txt',
                    'data': 'bar\n'
                },
                {
                    'path': 'yuki/coder/test_in/sample-3.txt',
                    'data': 'baz\n'
                },
            ],
            expected_values=[
                {
                    'path': 'yuki/coder/test_out/sample-2.txt',
                    'data': 'bar\n'
                },
                {
                    'path': 'yuki/coder/test_out/sample-3.txt',
                    'data': 'baz\n'
                },
            ],
        )

    def test_call_generate_output_format_hack(self):
        self.snippet_call_generate_output(
            args=['-c', cat(), '-d', 'a/b', '-f', 'c/test_%e/d/%s/e.case.txt'],
            input_files=[
                {
                    'path': 'a/b/c/test_in/d/sample.case.1/e.case.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'a/b/c/test_in/d/sample.case.2/e.case.txt',
                    'data': 'bar\n'
                },
            ],
            expected_values=[
                {
                    'path': 'a/b/c/test_out/d/sample.case.1/e.case.txt',
                    'data': 'foo\n'
                },
                {
                    'path': 'a/b/c/test_out/d/sample.case.2/e.case.txt',
                    'data': 'bar\n'
                },
            ],
        )

    def test_call_generate_output_in_parallel(self):
        TOTAL = 1000
        PARALLEL = 256
        input_files = []
        expected_values = []
        for i in range(TOTAL):
            name = 'sample-%03d' % i
            input_files += [{
                'path': 'test/{}.in'.format(name),
                'data': str(i),
            }]
            if i > TOTAL * 0.95:
                input_files += [{
                    'path': 'test/{}.out'.format(name),
                    'data': str(i),
                }]
            expected_values += [{
                'path': 'test/{}.in'.format(name),
                'data': str(i),
            }]
            expected_values += [{
                'path': 'test/{}.out'.format(name),
                'data': str(i),
            }]
        self.snippet_call_generate_output(
            args=['--jobs', str(PARALLEL), '-c', tests.utils.python_c("import sys, time; time.sleep(1); sys.stdout.write(sys.stdin.read())")],
            input_files=input_files,
            expected_values=expected_values,
        )
<|endoftext|>


<commit_before>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)
Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
from bs4 import BeautifulSoup
from collections import OrderedDict
import json
import pandas as pd
from datetime import datetime
from geopy.geocoders import Nominatim

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop']),
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        #TODO: Add __call__ option to redownload
        self.response = opener.open(self.url)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    def __call__(self):
        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        #TODO: Pull colNames ftom the table headngs
        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 0

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas Dataframe
        #Extract date and time colums to column of dateTime objects, then drop
        returnData = pd.DataFrame(dataset)
        returnData['DateTime'] = returnData.apply(lambda row datetime(row['Date'], row['Time'], axis = 1)
        returnData = returnData.drop(['Date', 'Time'], 1)

        #Location coersion
        geolocator = Nominatim()
        returnData['Location'] = returnData.apply(lambda row geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        returnData['Latitude'] = returnData.apply(lambda row geolocator.geocode(row['Location']).latitude)
        returnData['Longitude'] = returnData.apply(lambda row geolocator.geocode(row['Location']).longitude)

        #Ship it!
        return returnData
<commit_msg>Fixed some of the comments to bv<commit_after>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)
Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
from bs4 import BeautifulSoup
from collections import OrderedDict
import json
import pandas as pd
from datetime import datetime
from geopy.geocoders import Nominatim

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop']),
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        #TODO: Add __call__ option to redownload
        self.response = opener.open(self.url)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    #Setting this default method allows one to call an instance like a function. NEATO!
    def __call__(self):
        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        #TODO: Pull colNames from the table headngs
        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 0

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas DataFrame
        #Extract date and time colums to column of dateTime objects, then drop
        returnData = pd.DataFrame(dataset)
        returnData['DateTime'] = returnData.apply(lambda row datetime(row['Date'], row['Time'], axis = 1)
        returnData = returnData.drop(['Date', 'Time'], 1)

        #Location coersion
        geolocator = Nominatim()
        returnData['Location'] = returnData.apply(lambda row geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        returnData['Latitude'] = returnData.apply(lambda row geolocator.geocode(row['Location']).latitude)
        returnData['Longitude'] = returnData.apply(lambda row geolocator.geocode(row['Location']).longitude)

        #Ship it!
        return returnData
<|endoftext|>


<commit_before>#   This code has to be added to the corresponding __init__.py

DRIVERS["sensormock"] = ["PRESSURE", "TEMPERATURE", "LUMINOSITY", "DISTANCE", "HUMIDITY",
                         "COLOR", "CURRENT", "VOLTAGE", "POWER",
                         "LINEARACCELERATION", "ANGULARACCELERATION", "ACCELERATION", "LINEARVELOCITY", "ANGULARVELOCITY", "VELOCITY",
                         "SENSORS"]
<commit_msg>Delete drivers.py<commit_after><|endoftext|>


<commit_before><<<<<<< HEAD:metapdf/metapdf.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
.. module:: metapdf
   :platform: Unix, Windows
   :synopsis: The metaPdf implementation.

.. moduleauthor:: Ali Anari <ali@alianari.com>

"""

import os, re

class _meta_pdf_reader(object):

    def __init__(self):
        self.instance = self.__hash__()
        self.metadata_regex = re.compile('(?:\/(\w+)\s?\(([^\n\r]*)\)\n?\r?)', re.S)
        self.metadata_offset = 2048
        self.start_header = 10240 / self.metadata_offset

    def _metadata(self, f, depth, multiplier):

        """This is a helper method called by read_metadata.

        """

        lastindex = depth/multiplier
        f.seek((depth * -self.metadata_offset), os.SEEK_END)
        for i in range(1, lastindex):
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(multiplier * self.metadata_offset)))
            if properties: break
        return properties

    def read_metadata(self, file_name):

        """This function reads a PDF file and returns its metadata.
        :param file_name: The PDF file name to read.
        :type file_name: str
        :returns: dict -- The returned metadata as a dictionary of properties.

        """

        with open(str(file_name), 'rb') as f:

            # Scan the last 2048 bytes, the most
            # frequent metadata density block
            f.seek(-self.metadata_offset, os.SEEK_END)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset)))
            if properties:
                return properties

            # Scan forward from the last 200 * 2048
            # bytes, the next high density region
            properties = self._metadata(f, 200, 10)
            if properties:
                return properties

            # Investigate the header at 5 * 2048
            f.seek(0)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.start_header * self.metadata_offset)))
            if properties:
                return properties

            # Do a fast scan through the last 1000 * 2048
            properties = self._metadata(f, 800, 50)
            if properties:
                return properties

            # Last ditch attempt, scan entire file
            f.seek(start_header)
            while True:
                properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset*4)))
                if properties:
                    return properties

        return {}

_metaPdfReader = _meta_pdf_reader()
def MetaPdfReader(): return _metaPdfReader
=======
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
.. module:: metapdf
   :platform: Unix, Windows
   :synopsis: The metaPdf implementation.

.. moduleauthor:: Ali Anari <ali@alianari.com>

"""

import os, re

class _meta_pdf_reader(object):

    def __init__(self):
        self.instance = self.__hash__()
        self.metadata_regex = re.compile('(?:\/(\w+)\s?\(([^\n\r]*)\)\n?\r?)', re.S)
        self.metadata_offset = 2048
        self.start_header = 10240 / self.metadata_offset

    def _metadata(self, f, depth, multiplier):

        """This is a helper method called by read_metadata.

        """

        lastindex = depth/multiplier
        f.seek((depth * -self.metadata_offset), os.SEEK_END)
        for i in range(1, lastindex):
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(multiplier * self.metadata_offset)))
            if properties: break
        return properties

    def read_metadata(self, file_name):

        """This function reads a PDF file and returns its metadata.
        :param file_name: The PDF file name to read.
        :type file_name: str
        :returns: dict -- The returned metadata as a dictionary of properties.

        """

        with open(str(file_name), 'rb') as f:

            # Scan the last 2048 bytes, the most
            # frequent metadata density block
            f.seek(-self.metadata_offset, os.SEEK_END)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset)))
            if properties:
                return properties

            # Scan forward from the last 200 * 2048
            # bytes, the next high density region
            properties = self._metadata(f, 200, 10)
            if properties:
                return properties

            # Investigate the header at 5 * 2048
            f.seek(0)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.start_header * self.metadata_offset)))
            if properties:
                return properties

            # Do a fast scan through the last 1000 * 2048
            properties = self._metadata(f, 800, 50)
            if properties:
                return properties

            # Last ditch attempt, scan entire file
            f.seek(start_header)
            while True:
                properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset*4)))
                if properties:
                    return properties

        return {}

_metaPdfReader = _meta_pdf_reader()
def MetaPdfReader(): return _metaPdfReader
>>>>>>> b28a08ca21b014e15811b02610958e3d5efd1855:metaPdf/metaPdf.py
<commit_msg>Fixed comment and cleaned MANIFEST<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
.. module:: metapdf
   :platform: Unix, Windows
   :synopsis: The metapdf class implementation.

.. moduleauthor:: Ali Anari <ali@alianari.com>

"""

import os, re

class _meta_pdf_reader(object):

    def __init__(self):
        self.instance = self.__hash__()
        self.metadata_regex = re.compile('(?:\/(\w+)\s?\(([^\n\r]*)\)\n?\r?)', re.S)
        self.metadata_offset = 2048
        self.start_header = 10240 / self.metadata_offset

    def _metadata(self, f, depth, multiplier):

        """This is a helper method called by read_metadata.

        """

        lastindex = depth/multiplier
        f.seek((depth * -self.metadata_offset), os.SEEK_END)
        for i in range(1, lastindex):
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(multiplier * self.metadata_offset)))
            if properties: break
        return properties

    def read_metadata(self, file_name):

        """This function reads a PDF file and returns its metadata.
        :param file_name: The PDF file name to read.
        :type file_name: str
        :returns: dict -- The returned metadata as a dictionary of properties.

        """

        with open(str(file_name), 'rb') as f:

            # Scan the last 2048 bytes, the most
            # frequent metadata density block
            f.seek(-self.metadata_offset, os.SEEK_END)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset)))
            if properties:
                return properties

            # Scan forward from the last 200 * 2048
            # bytes, the next high density region
            properties = self._metadata(f, 200, 10)
            if properties:
                return properties

            # Investigate the header at 5 * 2048
            f.seek(0)
            properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.start_header * self.metadata_offset)))
            if properties:
                return properties

            # Do a fast scan through the last 1000 * 2048
            properties = self._metadata(f, 800, 50)
            if properties:
                return properties

            # Last ditch attempt, scan entire file
            f.seek(start_header)
            while True:
                properties = dict((p.group(1), p.group(2)) for p in self.metadata_regex.finditer(f.read(self.metadata_offset*4)))
                if properties:
                    return properties

        return {}

_metaPdfReader = _meta_pdf_reader()
def MetaPdfReader(): return _metaPdfReader
<|endoftext|>


<commit_before>'''
A simple way of setting the output format for data from modules
'''
# Import Python libs
import yaml
import pprint

# Conditionally import the json module
try:
    import json
    JSON = True
except ImportError:
    JSON = False

# Import Salt libs
import salt.utils

__all__ = ('get_outputter',)

def remove_colors():
    '''
    Acces all of the utility colors and change them to empy strings
    '''

class Outputter(object):
    '''
    Class for outputting data to the screen.
    '''
    supports = None

    @classmethod
    def check(klass, name):
       # Don't advertise Outputter classes for optional modules
       if hasattr(klass, "enabled") and not klass.enabled:
           return False
       return klass.supports == name

    def __call__(self, data, **kwargs):
        pprint.pprint(data)

class HighStateOutputter(Outputter):
    '''
    Not a command line option, the HighStateOutputter is only meant to be used
    with the state.highstate function, or a function that returns highstate
    return data
    '''
    supports = 'highstate'
    def __call__(self, data, **kwargs):
        colors = salt.utils.get_colors(kwargs.get('color'))
        for host in data:
            hcolor = colors['GREEN']
            hstrs = []
            if isinstance(data[host], list):
                hcolor = colors['RED_BOLD']
                hstrs.append('    {0}Data failed to compile:{1[ENDC]}'.format(hcolor, colors))
                for err in data[host]:
                    hstrs.append('    {0}{1}{2[ENDC]}'.format(hcolor, err, colors))
            if isinstance(data[host], dict):
                for tname, ret in data[host].items():
                    tcolor = colors['GREEN']
                    if not ret['result']:
                        hcolor = colors['RED']
                        tcolor = colors['RED']
                    comps = tname.split('.')
                    hstrs.append('{0}----------\n    State: - {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[0],
                        colors
                        ))
                    hstrs.append('    {0}Name:      {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[1],
                        colors
                        ))
                    hstrs.append('    {0}Function:  {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[2],
                        colors
                        ))
                    hstrs.append('        {0}Result:    {1}{2[ENDC]}'.format(
                        tcolor,
                        str(ret['result']),
                        colors
                        ))
                    hstrs.append('        {0}Comment:   {1}{2[ENDC]}'.format(
                        tcolor,
                        ret['comment'],
                        colors
                        ))
                    changes = '        Changes:   '
                    for key in ret['changes']:
                        if isinstance(ret['changes'][key], str):
                            changes += key + ': ' + ret['changes'][key] + '\n                   '
                        elif isinstance(ret['changes'][key], dict):
                            changes += key + ': ' + pprint.pformat(ret['changes'][key]) + '\n                   '
                        else:
                            changes += key + ': ' + pprint.pformat(ret['changes'][key]) + '\n                   '
                    hstrs.append('{0}{1}{2[ENDC]}'.format(
                        tcolor,
                        changes,
                        colors
                        ))
            print '{0}{1}:{2[ENDC]}'.format(
                hcolor,
                host,
                colors)
            for hstr in hstrs:
                print hstr

class RawOutputter(Outputter):
    '''
    Raw output. This calls repr() on the returned data.
    '''
    supports = "raw"
    def __call__(self, data, **kwargs):
        print data

class TxtOutputter(Outputter):
    '''
    Plain text output. Primarily for returning output from
    shell commands in the exact same way they would output
    on the shell when ran directly.
    '''
    supports = "txt"
    def __call__(self, data, **kwargs):
        if hasattr(data, "keys"):
            for key in data.keys():
                value = data[key]
                for line in value.split('\n'):
                    print "{0}: {1}".format(key, line)
        else:
            # For non-dictionary data, just use print
            RawOutputter()(data)

class JSONOutputter(Outputter):
    '''
    JSON output.
    '''
    supports = "json"
    enabled  = JSON

    def __call__(self, data, **kwargs):
        try:
            # A good kwarg might be: indent=4
            ret = json.dumps(data, **kwargs)
        except TypeError:
            # Return valid json for unserializable objects
            ret = json.dumps({})
        print ret

class YamlOutputter(Outputter):
    '''
    Yaml output. All of the cool kids are doing it.
    '''
    supports = "yaml"

    def __call__(self, data,  **kwargs):
        print yaml.dump(data, **kwargs)

def get_outputter(name=None):
    '''
    Factory function for returning the right output class.

    Usage:
        printout = get_outputter("txt")
        printout(ret)
    '''
    # Return an actual instance of the correct output class
    for i in Outputter.__subclasses__():
        if i.check(name):
            return i()
    return Outputter()
<commit_msg>make error output a little cleaner<commit_after>'''
A simple way of setting the output format for data from modules
'''
# Import Python libs
import yaml
import pprint

# Conditionally import the json module
try:
    import json
    JSON = True
except ImportError:
    JSON = False

# Import Salt libs
import salt.utils

__all__ = ('get_outputter',)

def remove_colors():
    '''
    Acces all of the utility colors and change them to empy strings
    '''

class Outputter(object):
    '''
    Class for outputting data to the screen.
    '''
    supports = None

    @classmethod
    def check(klass, name):
       # Don't advertise Outputter classes for optional modules
       if hasattr(klass, "enabled") and not klass.enabled:
           return False
       return klass.supports == name

    def __call__(self, data, **kwargs):
        pprint.pprint(data)

class HighStateOutputter(Outputter):
    '''
    Not a command line option, the HighStateOutputter is only meant to be used
    with the state.highstate function, or a function that returns highstate
    return data
    '''
    supports = 'highstate'
    def __call__(self, data, **kwargs):
        colors = salt.utils.get_colors(kwargs.get('color'))
        for host in data:
            hcolor = colors['GREEN']
            hstrs = []
            if isinstance(data[host], list):
                # Errors have been detected, list them in RED!
                hcolor = colors['RED_BOLD']
                hstrs.append('    {0}Data failed to compile:{1[ENDC]}'.format(hcolor, colors))
                for err in data[host]:
                    hstrs.append('{0}----------\n    {1}{2[ENDC]}'.format(hcolor, err, colors))
            if isinstance(data[host], dict):
                # Everything rendered as it should display the output
                for tname, ret in data[host].items():
                    tcolor = colors['GREEN']
                    if not ret['result']:
                        hcolor = colors['RED']
                        tcolor = colors['RED']
                    comps = tname.split('.')
                    hstrs.append('{0}----------\n    State: - {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[0],
                        colors
                        ))
                    hstrs.append('    {0}Name:      {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[1],
                        colors
                        ))
                    hstrs.append('    {0}Function:  {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[2],
                        colors
                        ))
                    hstrs.append('        {0}Result:    {1}{2[ENDC]}'.format(
                        tcolor,
                        str(ret['result']),
                        colors
                        ))
                    hstrs.append('        {0}Comment:   {1}{2[ENDC]}'.format(
                        tcolor,
                        ret['comment'],
                        colors
                        ))
                    changes = '        Changes:   '
                    for key in ret['changes']:
                        if isinstance(ret['changes'][key], str):
                            changes += key + ': ' + ret['changes'][key] + '\n                   '
                        elif isinstance(ret['changes'][key], dict):
                            changes += key + ': ' + pprint.pformat(ret['changes'][key]) + '\n                   '
                        else:
                            changes += key + ': ' + pprint.pformat(ret['changes'][key]) + '\n                   '
                    hstrs.append('{0}{1}{2[ENDC]}'.format(
                        tcolor,
                        changes,
                        colors
                        ))
            print '{0}{1}:{2[ENDC]}'.format(
                hcolor,
                host,
                colors)
            for hstr in hstrs:
                print hstr

class RawOutputter(Outputter):
    '''
    Raw output. This calls repr() on the returned data.
    '''
    supports = "raw"
    def __call__(self, data, **kwargs):
        print data

class TxtOutputter(Outputter):
    '''
    Plain text output. Primarily for returning output from
    shell commands in the exact same way they would output
    on the shell when ran directly.
    '''
    supports = "txt"
    def __call__(self, data, **kwargs):
        if hasattr(data, "keys"):
            for key in data.keys():
                value = data[key]
                for line in value.split('\n'):
                    print "{0}: {1}".format(key, line)
        else:
            # For non-dictionary data, just use print
            RawOutputter()(data)

class JSONOutputter(Outputter):
    '''
    JSON output.
    '''
    supports = "json"
    enabled  = JSON

    def __call__(self, data, **kwargs):
        try:
            # A good kwarg might be: indent=4
            ret = json.dumps(data, **kwargs)
        except TypeError:
            # Return valid json for unserializable objects
            ret = json.dumps({})
        print ret

class YamlOutputter(Outputter):
    '''
    Yaml output. All of the cool kids are doing it.
    '''
    supports = "yaml"

    def __call__(self, data,  **kwargs):
        print yaml.dump(data, **kwargs)

def get_outputter(name=None):
    '''
    Factory function for returning the right output class.

    Usage:
        printout = get_outputter("txt")
        printout(ret)
    '''
    # Return an actual instance of the correct output class
    for i in Outputter.__subclasses__():
        if i.check(name):
            return i()
    return Outputter()
<|endoftext|>


<commit_before># (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from ansible import errors
from ansible import utils
import os
import ansible.utils.template as template
import sys

class Task(object):

    __slots__ = [
        'name', 'meta', 'action', 'when', 'async_seconds', 'async_poll_interval',
        'notify', 'module_name', 'module_args', 'module_vars', 'default_vars',
        'play', 'notified_by', 'tags', 'register', 'role_name',
        'delegate_to', 'first_available_file', 'ignore_errors',
        'local_action', 'transport', 'sudo', 'remote_user', 'sudo_user', 'sudo_pass',
        'items_lookup_plugin', 'items_lookup_terms', 'environment', 'args',
        'any_errors_fatal', 'changed_when', 'failed_when', 'always_run', 'delay', 'retries', 'until',
        'su', 'su_user', 'su_pass', 'no_log',
    ]

    # to prevent typos and such
    VALID_KEYS = [
         'name', 'meta', 'action', 'when', 'async', 'poll', 'notify',
         'first_available_file', 'include', 'tags', 'register', 'ignore_errors',
         'delegate_to', 'local_action', 'transport', 'remote_user', 'sudo', 'sudo_user',
         'sudo_pass', 'when', 'connection', 'environment', 'args',
         'any_errors_fatal', 'changed_when', 'failed_when', 'always_run', 'delay', 'retries', 'until',
         'su', 'su_user', 'su_pass', 'no_log',
    ]

    def __init__(self, play, ds, module_vars=None, default_vars=None, additional_conditions=None, role_name=None):
        ''' constructor loads from a task or handler datastructure '''

        # meta directives are used to tell things like ansible/playbook to run
        # operations like handler execution.  Meta tasks are not executed
        # normally.
        if 'meta' in ds:
            self.meta = ds['meta']
            self.tags = []
            return
        else:
            self.meta = None

        library = os.path.join(play.basedir, 'library')
        if os.path.exists(library):
            utils.plugins.module_finder.add_directory(library)

        for x in ds.keys():

            # code to allow for saying "modulename: args" versus "action: modulename args"
            if x in utils.plugins.module_finder:

                if 'action' in ds:
                    raise errors.AnsibleError("multiple actions specified in task %s" % (ds.get('name', ds['action'])))
                if isinstance(ds[x], dict):
                    if 'args' in ds:
                        raise errors.AnsibleError("can't combine args: and a dict for %s: in task %s" % (x, ds.get('name', "%s: %s" % (x, ds[x]))))
                    ds['args'] = ds[x]
                    ds[x] = ''
                elif ds[x] is None:
                    ds[x] = ''
                if not isinstance(ds[x], basestring):
                    raise errors.AnsibleError("action specified for task %s has invalid type %s" % (ds.get('name', "%s: %s" % (x, ds[x])), type(ds[x])))
                ds['action'] = x + " " + ds[x]
                ds.pop(x)

            # code to allow "with_glob" and to reference a lookup plugin named glob
            elif x.startswith("with_"):

                if isinstance(ds[x], basestring) and ds[x].lstrip().startswith("{{"):
                    utils.warning("It is unnecessary to use '{{' in loops, leave variables in loop expressions bare.")

                plugin_name = x.replace("with_","")
                if plugin_name in utils.plugins.lookup_loader:
                    ds['items_lookup_plugin'] = plugin_name
                    ds['items_lookup_terms'] = ds[x]
                    ds.pop(x)
                else:
                    raise errors.AnsibleError("cannot find lookup plugin named %s for usage in with_%s" % (plugin_name, plugin_name))

            elif x in [ 'changed_when', 'failed_when', 'when']:
                if isinstance(ds[x], basestring) and ds[x].lstrip().startswith("{{"):
                    utils.warning("It is unnecessary to use '{{' in conditionals, leave variables in loop expressions bare.")
            elif x.startswith("when_"):
                utils.deprecated("The 'when_' conditional has been removed. Switch to using the regular unified 'when' statements as described on docs.ansible.com.","1.5", removed=True)

                if 'when' in ds:
                    raise errors.AnsibleError("multiple when_* statements specified in task %s" % (ds.get('name', ds['action'])))
                when_name = x.replace("when_","")
                ds['when'] = "%s %s" % (when_name, ds[x])
                ds.pop(x)
            elif not x in Task.VALID_KEYS:
                raise errors.AnsibleError("%s is not a legal parameter in an Ansible task or handler" % x)

        self.module_vars  = module_vars
        self.default_vars = default_vars
        self.play         = play

        # load various attributes
        self.name         = ds.get('name', None)
        self.tags         = [ 'all' ]
        self.register     = ds.get('register', None)
        self.sudo         = utils.boolean(ds.get('sudo', play.sudo))
        self.su           = utils.boolean(ds.get('su', play.su))
        self.environment  = ds.get('environment', {})
        self.role_name    = role_name
        self.no_log       = utils.boolean(ds.get('no_log', "false"))

        #Code to allow do until feature in a Task
        if 'until' in ds:
            if not ds.get('register'):
                raise errors.AnsibleError("register keyword is mandatory when using do until feature")
            self.module_vars['delay']     = ds.get('delay', 5)
            self.module_vars['retries']   = ds.get('retries', 3)
            self.module_vars['register']  = ds.get('register', None)
            self.until                    = ds.get('until')
            self.module_vars['until']     = self.until

        # rather than simple key=value args on the options line, these represent structured data and the values
        # can be hashes and lists, not just scalars
        self.args         = ds.get('args', {})

        # get remote_user for task, then play, then playbook
        if ds.get('remote_user') is not None:
            self.remote_user      = ds.get('remote_user')
        elif ds.get('remote_user', play.remote_user) is not None:
            self.remote_user      = ds.get('remote_user', play.remote_user)
        else:
            self.remote_user      = ds.get('remote_user', play.playbook.remote_user)

        self.sudo_user    = None
        self.sudo_pass    = None
        self.su_user      = None
        self.su_pass      = None

        if self.sudo:
            self.sudo_user    = ds.get('sudo_user', play.sudo_user)
            self.sudo_pass    = ds.get('sudo_pass', play.playbook.sudo_pass)
        elif self.su:
            self.su_user      = ds.get('su_user', play.su_user)
            self.su_pass      = ds.get('su_pass', play.playbook.su_pass)

        # Fail out if user specifies a sudo param with a su param in a given play
        if (ds.get('sudo') or ds.get('sudo_user') or ds.get('sudo_pass')) and \
                (ds.get('su') or ds.get('su_user') or ds.get('su_pass')):
            raise errors.AnsibleError('sudo params ("sudo", "sudo_user", "sudo_pass") '
                                      'and su params "su", "su_user", "su_pass") '
                                      'cannot be used together')

        # Both are defined
        if ('action' in ds) and ('local_action' in ds):
            raise errors.AnsibleError("the 'action' and 'local_action' attributes can not be used together")
        # Both are NOT defined
        elif (not 'action' in ds) and (not 'local_action' in ds):
            raise errors.AnsibleError("'action' or 'local_action' attribute missing in task \"%s\"" % ds.get('name', '<Unnamed>'))
        # Only one of them is defined
        elif 'local_action' in ds:
            self.action      = ds.get('local_action', '')
            self.delegate_to = '127.0.0.1'
        else:
            self.action      = ds.get('action', '')
            self.delegate_to = ds.get('delegate_to', None)
            self.transport   = ds.get('connection', ds.get('transport', play.transport))

        if isinstance(self.action, dict):
            if 'module' not in self.action:
                raise errors.AnsibleError("'module' attribute missing from action in task \"%s\"" % ds.get('name', '%s' % self.action))
            if self.args:
                raise errors.AnsibleError("'args' cannot be combined with dict 'action' in task \"%s\"" % ds.get('name', '%s' % self.action))
            self.args = self.action
            self.action = self.args.pop('module')

        # delegate_to can use variables
        if not (self.delegate_to is None):
            # delegate_to: localhost should use local transport
            if self.delegate_to in ['127.0.0.1', 'localhost']:
                self.transport   = 'local'

        # notified by is used by Playbook code to flag which hosts
        # need to run a notifier
        self.notified_by = []

        # if no name is specified, use the action line as the name
        if self.name is None:
            self.name = self.action

        # load various attributes
        self.when    = ds.get('when', None)
        self.changed_when = ds.get('changed_when', None)
        self.failed_when = ds.get('failed_when', None)

        self.async_seconds = ds.get('async', 0)  # not async by default
        self.async_seconds = template.template_from_string(play.basedir, self.async_seconds, self.module_vars)
        self.async_seconds = int(self.async_seconds)
        self.async_poll_interval = ds.get('poll', 10)  # default poll = 10 seconds
        self.async_poll_interval = template.template_from_string(play.basedir, self.async_poll_interval, self.module_vars)
        self.async_poll_interval = int(self.async_poll_interval)
        self.notify = ds.get('notify', [])
        self.first_available_file = ds.get('first_available_file', None)

        self.items_lookup_plugin = ds.get('items_lookup_plugin', None)
        self.items_lookup_terms  = ds.get('items_lookup_terms', None)

        self.ignore_errors = ds.get('ignore_errors', False)
        self.any_errors_fatal = ds.get('any_errors_fatal', play.any_errors_fatal)

        self.always_run = ds.get('always_run', False)

        # action should be a string
        if not isinstance(self.action, basestring):
            raise errors.AnsibleError("action is of type '%s' and not a string in task. name: %s" % (type(self.action).__name__, self.name))

        # notify can be a string or a list, store as a list
        if isinstance(self.notify, basestring):
            self.notify = [ self.notify ]

        # split the action line into a module name + arguments
        tokens = self.action.split(None, 1)
        if len(tokens) < 1:
            raise errors.AnsibleError("invalid/missing action in task. name: %s" % self.name)
        self.module_name = tokens[0]
        self.module_args = ''
        if len(tokens) > 1:
            self.module_args = tokens[1]

        import_tags = self.module_vars.get('tags',[])
        if type(import_tags) in [int,float]:
            import_tags = str(import_tags)
        elif type(import_tags) in [str,unicode]:
            # allow the user to list comma delimited tags
            import_tags = import_tags.split(",")

        # handle mutually incompatible options
        incompatibles = [ x for x in [ self.first_available_file, self.items_lookup_plugin ] if x is not None ]
        if len(incompatibles) > 1:
            raise errors.AnsibleError("with_(plugin), and first_available_file are mutually incompatible in a single task")

        # make first_available_file accessable to Runner code
        if self.first_available_file:
            self.module_vars['first_available_file'] = self.first_available_file

        if self.items_lookup_plugin is not None:
            self.module_vars['items_lookup_plugin'] = self.items_lookup_plugin
            self.module_vars['items_lookup_terms'] = self.items_lookup_terms

        # allow runner to see delegate_to option
        self.module_vars['delegate_to'] = self.delegate_to

        # make some task attributes accessible to Runner code
        self.module_vars['ignore_errors'] = self.ignore_errors
        self.module_vars['register'] = self.register
        self.module_vars['changed_when'] = self.changed_when
        self.module_vars['failed_when'] = self.failed_when
        self.module_vars['always_run'] = self.always_run

        # tags allow certain parts of a playbook to be run without running the whole playbook
        apply_tags = ds.get('tags', None)
        if apply_tags is not None:
            if type(apply_tags) in [ str, unicode ]:
                self.tags.append(apply_tags)
            elif type(apply_tags) in [ int, float ]:
                self.tags.append(str(apply_tags))
            elif type(apply_tags) == list:
                self.tags.extend(apply_tags)
        self.tags.extend(import_tags)

        if additional_conditions:
            new_conditions = additional_conditions
            new_conditions.append(self.when)
            self.when = new_conditions
<commit_msg>Improve error message when multiple action are given<commit_after># (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from ansible import errors
from ansible import utils
import os
import ansible.utils.template as template
import sys

class Task(object):

    __slots__ = [
        'name', 'meta', 'action', 'when', 'async_seconds', 'async_poll_interval',
        'notify', 'module_name', 'module_args', 'module_vars', 'default_vars',
        'play', 'notified_by', 'tags', 'register', 'role_name',
        'delegate_to', 'first_available_file', 'ignore_errors',
        'local_action', 'transport', 'sudo', 'remote_user', 'sudo_user', 'sudo_pass',
        'items_lookup_plugin', 'items_lookup_terms', 'environment', 'args',
        'any_errors_fatal', 'changed_when', 'failed_when', 'always_run', 'delay', 'retries', 'until',
        'su', 'su_user', 'su_pass', 'no_log',
    ]

    # to prevent typos and such
    VALID_KEYS = [
         'name', 'meta', 'action', 'when', 'async', 'poll', 'notify',
         'first_available_file', 'include', 'tags', 'register', 'ignore_errors',
         'delegate_to', 'local_action', 'transport', 'remote_user', 'sudo', 'sudo_user',
         'sudo_pass', 'when', 'connection', 'environment', 'args',
         'any_errors_fatal', 'changed_when', 'failed_when', 'always_run', 'delay', 'retries', 'until',
         'su', 'su_user', 'su_pass', 'no_log',
    ]

    def __init__(self, play, ds, module_vars=None, default_vars=None, additional_conditions=None, role_name=None):
        ''' constructor loads from a task or handler datastructure '''

        # meta directives are used to tell things like ansible/playbook to run
        # operations like handler execution.  Meta tasks are not executed
        # normally.
        if 'meta' in ds:
            self.meta = ds['meta']
            self.tags = []
            return
        else:
            self.meta = None

        library = os.path.join(play.basedir, 'library')
        if os.path.exists(library):
            utils.plugins.module_finder.add_directory(library)

        for x in ds.keys():

            # code to allow for saying "modulename: args" versus "action: modulename args"
            if x in utils.plugins.module_finder:

                if 'action' in ds:
                    raise errors.AnsibleError("multiple actions specified in task: '%s' and '%s'" % (x, ds.get('name', ds['action'])))
                if isinstance(ds[x], dict):
                    if 'args' in ds:
                        raise errors.AnsibleError("can't combine args: and a dict for %s: in task %s" % (x, ds.get('name', "%s: %s" % (x, ds[x]))))
                    ds['args'] = ds[x]
                    ds[x] = ''
                elif ds[x] is None:
                    ds[x] = ''
                if not isinstance(ds[x], basestring):
                    raise errors.AnsibleError("action specified for task %s has invalid type %s" % (ds.get('name', "%s: %s" % (x, ds[x])), type(ds[x])))
                ds['action'] = x + " " + ds[x]
                ds.pop(x)

            # code to allow "with_glob" and to reference a lookup plugin named glob
            elif x.startswith("with_"):

                if isinstance(ds[x], basestring) and ds[x].lstrip().startswith("{{"):
                    utils.warning("It is unnecessary to use '{{' in loops, leave variables in loop expressions bare.")

                plugin_name = x.replace("with_","")
                if plugin_name in utils.plugins.lookup_loader:
                    ds['items_lookup_plugin'] = plugin_name
                    ds['items_lookup_terms'] = ds[x]
                    ds.pop(x)
                else:
                    raise errors.AnsibleError("cannot find lookup plugin named %s for usage in with_%s" % (plugin_name, plugin_name))

            elif x in [ 'changed_when', 'failed_when', 'when']:
                if isinstance(ds[x], basestring) and ds[x].lstrip().startswith("{{"):
                    utils.warning("It is unnecessary to use '{{' in conditionals, leave variables in loop expressions bare.")
            elif x.startswith("when_"):
                utils.deprecated("The 'when_' conditional has been removed. Switch to using the regular unified 'when' statements as described on docs.ansible.com.","1.5", removed=True)

                if 'when' in ds:
                    raise errors.AnsibleError("multiple when_* statements specified in task %s" % (ds.get('name', ds['action'])))
                when_name = x.replace("when_","")
                ds['when'] = "%s %s" % (when_name, ds[x])
                ds.pop(x)
            elif not x in Task.VALID_KEYS:
                raise errors.AnsibleError("%s is not a legal parameter in an Ansible task or handler" % x)

        self.module_vars  = module_vars
        self.default_vars = default_vars
        self.play         = play

        # load various attributes
        self.name         = ds.get('name', None)
        self.tags         = [ 'all' ]
        self.register     = ds.get('register', None)
        self.sudo         = utils.boolean(ds.get('sudo', play.sudo))
        self.su           = utils.boolean(ds.get('su', play.su))
        self.environment  = ds.get('environment', {})
        self.role_name    = role_name
        self.no_log       = utils.boolean(ds.get('no_log', "false"))

        #Code to allow do until feature in a Task
        if 'until' in ds:
            if not ds.get('register'):
                raise errors.AnsibleError("register keyword is mandatory when using do until feature")
            self.module_vars['delay']     = ds.get('delay', 5)
            self.module_vars['retries']   = ds.get('retries', 3)
            self.module_vars['register']  = ds.get('register', None)
            self.until                    = ds.get('until')
            self.module_vars['until']     = self.until

        # rather than simple key=value args on the options line, these represent structured data and the values
        # can be hashes and lists, not just scalars
        self.args         = ds.get('args', {})

        # get remote_user for task, then play, then playbook
        if ds.get('remote_user') is not None:
            self.remote_user      = ds.get('remote_user')
        elif ds.get('remote_user', play.remote_user) is not None:
            self.remote_user      = ds.get('remote_user', play.remote_user)
        else:
            self.remote_user      = ds.get('remote_user', play.playbook.remote_user)

        self.sudo_user    = None
        self.sudo_pass    = None
        self.su_user      = None
        self.su_pass      = None

        if self.sudo:
            self.sudo_user    = ds.get('sudo_user', play.sudo_user)
            self.sudo_pass    = ds.get('sudo_pass', play.playbook.sudo_pass)
        elif self.su:
            self.su_user      = ds.get('su_user', play.su_user)
            self.su_pass      = ds.get('su_pass', play.playbook.su_pass)

        # Fail out if user specifies a sudo param with a su param in a given play
        if (ds.get('sudo') or ds.get('sudo_user') or ds.get('sudo_pass')) and \
                (ds.get('su') or ds.get('su_user') or ds.get('su_pass')):
            raise errors.AnsibleError('sudo params ("sudo", "sudo_user", "sudo_pass") '
                                      'and su params "su", "su_user", "su_pass") '
                                      'cannot be used together')

        # Both are defined
        if ('action' in ds) and ('local_action' in ds):
            raise errors.AnsibleError("the 'action' and 'local_action' attributes can not be used together")
        # Both are NOT defined
        elif (not 'action' in ds) and (not 'local_action' in ds):
            raise errors.AnsibleError("'action' or 'local_action' attribute missing in task \"%s\"" % ds.get('name', '<Unnamed>'))
        # Only one of them is defined
        elif 'local_action' in ds:
            self.action      = ds.get('local_action', '')
            self.delegate_to = '127.0.0.1'
        else:
            self.action      = ds.get('action', '')
            self.delegate_to = ds.get('delegate_to', None)
            self.transport   = ds.get('connection', ds.get('transport', play.transport))

        if isinstance(self.action, dict):
            if 'module' not in self.action:
                raise errors.AnsibleError("'module' attribute missing from action in task \"%s\"" % ds.get('name', '%s' % self.action))
            if self.args:
                raise errors.AnsibleError("'args' cannot be combined with dict 'action' in task \"%s\"" % ds.get('name', '%s' % self.action))
            self.args = self.action
            self.action = self.args.pop('module')

        # delegate_to can use variables
        if not (self.delegate_to is None):
            # delegate_to: localhost should use local transport
            if self.delegate_to in ['127.0.0.1', 'localhost']:
                self.transport   = 'local'

        # notified by is used by Playbook code to flag which hosts
        # need to run a notifier
        self.notified_by = []

        # if no name is specified, use the action line as the name
        if self.name is None:
            self.name = self.action

        # load various attributes
        self.when    = ds.get('when', None)
        self.changed_when = ds.get('changed_when', None)
        self.failed_when = ds.get('failed_when', None)

        self.async_seconds = ds.get('async', 0)  # not async by default
        self.async_seconds = template.template_from_string(play.basedir, self.async_seconds, self.module_vars)
        self.async_seconds = int(self.async_seconds)
        self.async_poll_interval = ds.get('poll', 10)  # default poll = 10 seconds
        self.async_poll_interval = template.template_from_string(play.basedir, self.async_poll_interval, self.module_vars)
        self.async_poll_interval = int(self.async_poll_interval)
        self.notify = ds.get('notify', [])
        self.first_available_file = ds.get('first_available_file', None)

        self.items_lookup_plugin = ds.get('items_lookup_plugin', None)
        self.items_lookup_terms  = ds.get('items_lookup_terms', None)

        self.ignore_errors = ds.get('ignore_errors', False)
        self.any_errors_fatal = ds.get('any_errors_fatal', play.any_errors_fatal)

        self.always_run = ds.get('always_run', False)

        # action should be a string
        if not isinstance(self.action, basestring):
            raise errors.AnsibleError("action is of type '%s' and not a string in task. name: %s" % (type(self.action).__name__, self.name))

        # notify can be a string or a list, store as a list
        if isinstance(self.notify, basestring):
            self.notify = [ self.notify ]

        # split the action line into a module name + arguments
        tokens = self.action.split(None, 1)
        if len(tokens) < 1:
            raise errors.AnsibleError("invalid/missing action in task. name: %s" % self.name)
        self.module_name = tokens[0]
        self.module_args = ''
        if len(tokens) > 1:
            self.module_args = tokens[1]

        import_tags = self.module_vars.get('tags',[])
        if type(import_tags) in [int,float]:
            import_tags = str(import_tags)
        elif type(import_tags) in [str,unicode]:
            # allow the user to list comma delimited tags
            import_tags = import_tags.split(",")

        # handle mutually incompatible options
        incompatibles = [ x for x in [ self.first_available_file, self.items_lookup_plugin ] if x is not None ]
        if len(incompatibles) > 1:
            raise errors.AnsibleError("with_(plugin), and first_available_file are mutually incompatible in a single task")

        # make first_available_file accessable to Runner code
        if self.first_available_file:
            self.module_vars['first_available_file'] = self.first_available_file

        if self.items_lookup_plugin is not None:
            self.module_vars['items_lookup_plugin'] = self.items_lookup_plugin
            self.module_vars['items_lookup_terms'] = self.items_lookup_terms

        # allow runner to see delegate_to option
        self.module_vars['delegate_to'] = self.delegate_to

        # make some task attributes accessible to Runner code
        self.module_vars['ignore_errors'] = self.ignore_errors
        self.module_vars['register'] = self.register
        self.module_vars['changed_when'] = self.changed_when
        self.module_vars['failed_when'] = self.failed_when
        self.module_vars['always_run'] = self.always_run

        # tags allow certain parts of a playbook to be run without running the whole playbook
        apply_tags = ds.get('tags', None)
        if apply_tags is not None:
            if type(apply_tags) in [ str, unicode ]:
                self.tags.append(apply_tags)
            elif type(apply_tags) in [ int, float ]:
                self.tags.append(str(apply_tags))
            elif type(apply_tags) == list:
                self.tags.extend(apply_tags)
        self.tags.extend(import_tags)

        if additional_conditions:
            new_conditions = additional_conditions
            new_conditions.append(self.when)
            self.when = new_conditions
<|endoftext|>


<commit_before>#! /usr/bin/python
"""
Copyright 2013, Big Switch Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.

circuitpusher utilizes floodlight rest APIs to create a bidirectional circuit,
i.e., permanent flow entry, on all switches in route between two devices based
on IP addresses with specified priority.

Notes:
 1. The circuit pusher currently only creates circuit with two IP end points
 2. Prior to sending restAPI requests to the circuit pusher, the specified end
    points must already been known to the controller (i.e., already have sent
    packets on the network, easy way to assure this is to do a ping (to any
    target) from the two hosts.
 3. The current supported command syntax format is:
    a) circuitpusher.py --controller={IP}:{rest port} --type ip --src {IP} --dst {IP} --add --name {circuit-name}
       circuitpusher.py --controller={IP}:{rest port} --type mac --src {MAC} --dst {MAC} --add --name {circuit-name}

       adds a new circuit between src and dst devices Currently ip and mac circuit is supported. ARP is automatically supported.

       Currently a simple circuit record storage is provided in a text file circuits.json in the working directory.
       The file is not protected and does not clean itself between controller restarts.  The file is needed for correct operation
       and the user should make sure deleting the file when floodlight controller is restarted.

    b) circuitpusher.py --controller={IP}:{rest port} --delete --name {circuit-name}

       deletes a created circuit (as recorded in circuits.json) using the previously given name

@author kcwang
"""

import os
import sys
import subprocess
import json
import argparse
import io
import time

# parse circuit options.  Currently supports add and delete actions.
# Syntax:
#   circuitpusher --controller {IP:REST_PORT} --add --name {CIRCUIT_NAME} --type ip --src {IP} --dst {IP}
#   circuitpusher --controller {IP:REST_PORT} --add --name {CIRCUIT_NAME} --type mac --src {MAC} --dst {MAC}
#   circuitpusher --controller {IP:REST_PORT} --delete --name {CIRCUIT_NAME}

parser = argparse.ArgumentParser(description='Circuit Pusher')
parser.add_argument('--controller', dest='controllerRestIp', action='store', default='localhost:8080', help='controller IP:RESTport, e.g., localhost:8080 or A.B.C.D:8080')
parser.add_argument('--add', dest='action', action='store_const', const='add', default='add', help='action: add, delete')
parser.add_argument('--delete', dest='action', action='store_const', const='delete', default='add', help='action: add, delete')
parser.add_argument('--type', dest='type', action='store', default='ip', help='valid types: ip, mac')
parser.add_argument('--src', dest='srcAddress', action='store', default='0.0.0.0', help='source address: if type=ip, A.B.C.D')
parser.add_argument('--dst', dest='dstAddress', action='store', default='0.0.0.0', help='destination address: if type=ip, A.B.C.D')
parser.add_argument('--name', dest='circuitName', action='store', default='circuit-1', help='name for circuit, e.g., circuit-1')

args = parser.parse_args()
print args

controllerRestIp = args.controllerRestIp

# first check if a local file exists, which needs to be updated after add/delete
if os.path.exists('./circuits.json'):
    circuitDb = open('./circuits.json','r')
    lines = circuitDb.readlines()
    circuitDb.close()
else:
    lines={}

if args.action=='add':

    circuitDb = open('./circuits.json','a')

    for line in lines:
        data = json.loads(line)
        if data['name']==(args.circuitName):
            print "Circuit %s exists already. Use new name to create." % args.circuitName
            sys.exit()
        else:
            circuitExists = False

    if args.type == 'ip':
      deviceType = 'ipv4'
    elif args.type == 'mac':
      deviceType = 'mac'
      if len(args.srcAddress) != 17 or len(args.dstAddress) != 17:
        print "ERROR : MAC address %s not in correct format. Use format 12:34:56:78:90:12." % args.srcAddress
        sys.exit()
      if len(args.srcAddress) != 17 or len(args.dstAddress) != 17:
        print "ERROR : MAC address %s not in correct format. Use format 12:34:56:78:90:12." % args.dstAddress
        sys.exit()
    else:
      print "ERROR : Type must be ip or mac."
      sys.exit()

    # retrieve source and destination device attachment points
    # using DeviceManager rest API

    command = "curl -s http://%s/wm/device/?%s=%s" % (args.controllerRestIp, deviceType, args.srcAddress)
    result = os.popen(command).read()
    parsedResult = json.loads(result)
    print command+"\n"

    try:
    	sourceSwitch = parsedResult[0]['attachmentPoint'][0]['switchDPID']
    except IndexError:
	print "ERROR : the specified end point (%s) must already been known to the controller (i.e., already have sent packets on the network, easy way to assure this is to do a ping (to any target) from the two hosts." % (args.srcAddress)
	sys.exit()

    sourcePort = parsedResult[0]['attachmentPoint'][0]['port']

    command = "curl -s http://%s/wm/device/?%s=%s" % (args.controllerRestIp, deviceType, args.dstAddress)
    result = os.popen(command).read()
    parsedResult = json.loads(result)
    print command+"\n"

    try:
        destSwitch = parsedResult[0]['attachmentPoint'][0]['switchDPID']
    except IndexError:
        print "ERROR : the specified end point (%s) must already been known to the controller (i.e., already have sent packets on the network, easy way to assure this is to do a ping (to any target) from the two hosts." % (args.dstAddress)
        sys.exit()

    destPort = parsedResult[0]['attachmentPoint'][0]['port']

    print "Creating circuit:"
    print "from source device at switch %s port %s" % (sourceSwitch,sourcePort)
    print "to destination device at switch %s port %s"% (destSwitch,destPort)

    # retrieving route from source to destination
    # using Routing rest API

    command = "curl -s http://%s/wm/topology/route/%s/%s/%s/%s/json" % (controllerRestIp, sourceSwitch, sourcePort, destSwitch, destPort)

    result = os.popen(command).read()
    parsedResult = json.loads(result)

    print command+"\n"
    print result+"\n"

    for i in range(len(parsedResult)):
        if i % 2 == 0:
            ap1Dpid = parsedResult[i]['switch']
            ap1Port = parsedResult[i]['port']
            print ap1Dpid, ap1Port

        else:
            ap2Dpid = parsedResult[i]['switch']
            ap2Port = parsedResult[i]['port']
            print ap2Dpid, ap2Port

            # send one flow mod per pair of APs in route
            # using StaticFlowPusher rest API

            # IMPORTANT NOTE: current Floodlight StaticflowEntryPusher
            # assumes all flow entries to have unique name across all switches
            # this will most possibly be relaxed later, but for now we
            # encode each flow entry's name with both switch dpid, user
            # specified name, and flow type (f: forward, r: reverse, farp/rarp: arp)

            if args.type == 'ip':

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-ip\":\"%s\", \"dst-ip\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".f", args.srcAddress, args.dstAddress, "0x800", ap1Port, ap2Port, controllerRestIp)
              result = os.popen(command).read()
              print command

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-ip\":\"%s\", \"dst-ip\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".r", args.dstAddress, args.srcAddress, "0x800", ap2Port, ap1Port, controllerRestIp)
              result = os.popen(command).read()
              print command

            elif args.type == 'mac':

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-mac\":\"%s\", \"dst-mac\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".f", args.srcAddress, args.dstAddress, ap1Port, ap2Port, controllerRestIp)
              result = os.popen(command).read()
              print command

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-mac\":\"%s\", \"dst-mac\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".r", args.dstAddress, args.srcAddress, ap2Port, ap1Port, controllerRestIp)
              result = os.popen(command).read()
              print command

            command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".farp", "0x806", ap1Port, ap2Port, controllerRestIp)
            result = os.popen(command).read()
            print command

            command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".rarp", "0x806", ap2Port, ap1Port, controllerRestIp)
            result = os.popen(command).read()
            print command

            # store created circuit attributes in local ./circuits.json
            datetime = time.asctime()
            circuitParams = {'name':args.circuitName, 'Dpid':ap1Dpid, 'inPort':ap1Port, 'outPort':ap2Port, 'datetime':datetime}
            str = json.dumps(circuitParams)
            circuitDb.write(str+"\n")

        # confirm successful circuit creation
        # using controller rest API

        command="curl -s http://%s/wm/core/switch/all/flow/json| python -mjson.tool" % (controllerRestIp)
        result = os.popen(command).read()
        print command + "\n" + result

elif args.action=='delete':

    circuitDb = open('./circuits.json','w')

    # removing previously created flow from switches
    # using StaticFlowPusher rest API
    # currently, circuitpusher records created circuits in local file ./circuits.db
    # with circuit name and list of switches

    circuitExists = False

    for line in lines:
        data = json.loads(line)
        if data['name']==(args.circuitName):
            circuitExists = True

            sw = data['Dpid']
            print data, sw

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".f", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".farp", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".r", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".rarp", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

        else:
            circuitDb.write(line)

    circuitDb.close()

    if not circuitExists:
        print "specified circuit does not exist"
        sys.exit()

<commit_msg>Add return codes != 1 for error conditions, print out more things.<commit_after>#! /usr/bin/python
"""
Copyright 2013, Big Switch Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.

circuitpusher utilizes floodlight rest APIs to create a bidirectional circuit,
i.e., permanent flow entry, on all switches in route between two devices based
on IP addresses with specified priority.

Notes:
 1. The circuit pusher currently only creates circuit with two IP end points
 2. Prior to sending restAPI requests to the circuit pusher, the specified end
    points must already been known to the controller (i.e., already have sent
    packets on the network, easy way to assure this is to do a ping (to any
    target) from the two hosts.
 3. The current supported command syntax format is:
    a) circuitpusher.py --controller={IP}:{rest port} --type ip --src {IP} --dst {IP} --add --name {circuit-name}
       circuitpusher.py --controller={IP}:{rest port} --type mac --src {MAC} --dst {MAC} --add --name {circuit-name}

       adds a new circuit between src and dst devices Currently ip and mac circuit is supported. ARP is automatically supported.

       Currently a simple circuit record storage is provided in a text file circuits.json in the working directory.
       The file is not protected and does not clean itself between controller restarts.  The file is needed for correct operation
       and the user should make sure deleting the file when floodlight controller is restarted.

    b) circuitpusher.py --controller={IP}:{rest port} --delete --name {circuit-name}

       deletes a created circuit (as recorded in circuits.json) using the previously given name

@author kcwang
"""

import os
import sys
import subprocess
import json
import argparse
import io
import time

# parse circuit options.  Currently supports add and delete actions.
# Syntax:
#   circuitpusher --controller {IP:REST_PORT} --add --name {CIRCUIT_NAME} --type ip --src {IP} --dst {IP}
#   circuitpusher --controller {IP:REST_PORT} --add --name {CIRCUIT_NAME} --type mac --src {MAC} --dst {MAC}
#   circuitpusher --controller {IP:REST_PORT} --delete --name {CIRCUIT_NAME}

parser = argparse.ArgumentParser(description='Circuit Pusher')
parser.add_argument('--controller', dest='controllerRestIp', action='store', default='localhost:8080', help='controller IP:RESTport, e.g., localhost:8080 or A.B.C.D:8080')
parser.add_argument('--add', dest='action', action='store_const', const='add', default='add', help='action: add, delete')
parser.add_argument('--delete', dest='action', action='store_const', const='delete', default='add', help='action: add, delete')
parser.add_argument('--type', dest='type', action='store', default='ip', help='valid types: ip, mac')
parser.add_argument('--src', dest='srcAddress', action='store', default='0.0.0.0', help='source address: if type=ip, A.B.C.D')
parser.add_argument('--dst', dest='dstAddress', action='store', default='0.0.0.0', help='destination address: if type=ip, A.B.C.D')
parser.add_argument('--name', dest='circuitName', action='store', default='circuit-1', help='name for circuit, e.g., circuit-1')

args = parser.parse_args()
print args

controllerRestIp = args.controllerRestIp

# first check if a local file exists, which needs to be updated after add/delete
if os.path.exists('./circuits.json'):
    circuitDb = open('./circuits.json','r')
    lines = circuitDb.readlines()
    circuitDb.close()
else:
    lines={}

if args.action=='add':

    circuitDb = open('./circuits.json','a')

    for line in lines:
        data = json.loads(line)
        if data['name']==(args.circuitName):
            print "Circuit %s exists already. Use new name to create." % args.circuitName
            sys.exit(1)
        else:
            circuitExists = False

    if args.type == 'ip':
      deviceType = 'ipv4'
    elif args.type == 'mac':
      deviceType = 'mac'
      if len(args.srcAddress) != 17 or len(args.dstAddress) != 17:
        print "ERROR : MAC address %s not in correct format. Use format 12:34:56:78:90:12." % args.srcAddress
        sys.exit(1)
      if len(args.srcAddress) != 17 or len(args.dstAddress) != 17:
        print "ERROR : MAC address %s not in correct format. Use format 12:34:56:78:90:12." % args.dstAddress
        sys.exit(1)
    else:
      print "ERROR : Type must be ip or mac."
      sys.exit(1)

    # retrieve source and destination device attachment points
    # using DeviceManager rest API

    command = "curl -s http://%s/wm/device/?%s=%s" % (args.controllerRestIp, deviceType, args.srcAddress)
    result = os.popen(command).read()
    parsedResult = json.loads(result)
    print command + "\n" + result + "\n"

    try:
    	sourceSwitch = parsedResult[0]['attachmentPoint'][0]['switchDPID']
    except IndexError:
    	print "ERROR : the specified end point (%s) must already been known to the controller (i.e., already have sent packets on the network, easy way to assure this is to do a ping (to any target) from the two hosts." % (args.srcAddress)
    	sys.exit(1)

    sourcePort = parsedResult[0]['attachmentPoint'][0]['port']

    command = "curl -s http://%s/wm/device/?%s=%s" % (args.controllerRestIp, deviceType, args.dstAddress)
    result = os.popen(command).read()
    parsedResult = json.loads(result)
    print command + "\n" + result + "\n"

    try:
      destSwitch = parsedResult[0]['attachmentPoint'][0]['switchDPID']
    except IndexError:
      print "ERROR : the specified end point (%s) must already been known to the controller (i.e., already have sent packets on the network, easy way to assure this is to do a ping (to any target) from the two hosts." % (args.dstAddress)
      sys.exit(1)

    destPort = parsedResult[0]['attachmentPoint'][0]['port']

    print "Creating circuit:"
    print "from source device at switch %s port %s" % (sourceSwitch,sourcePort)
    print "to destination device at switch %s port %s"% (destSwitch,destPort)

    # retrieving route from source to destination
    # using Routing rest API

    command = "curl -s http://%s/wm/topology/route/%s/%s/%s/%s/json" % (controllerRestIp, sourceSwitch, sourcePort, destSwitch, destPort)

    result = os.popen(command).read()
    parsedResult = json.loads(result)

    print command + "\n" + result + "\n"

    for i in range(len(parsedResult)):
        if i % 2 == 0:
            ap1Dpid = parsedResult[i]['switch']
            ap1Port = parsedResult[i]['port']
            print ap1Dpid, ap1Port

        else:
            ap2Dpid = parsedResult[i]['switch']
            ap2Port = parsedResult[i]['port']
            print ap2Dpid, ap2Port

            # send one flow mod per pair of APs in route
            # using StaticFlowPusher rest API

            # IMPORTANT NOTE: current Floodlight StaticflowEntryPusher
            # assumes all flow entries to have unique name across all switches
            # this will most possibly be relaxed later, but for now we
            # encode each flow entry's name with both switch dpid, user
            # specified name, and flow type (f: forward, r: reverse, farp/rarp: arp)

            if args.type == 'ip':

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-ip\":\"%s\", \"dst-ip\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".f", args.srcAddress, args.dstAddress, "0x800", ap1Port, ap2Port, controllerRestIp)
              result = os.popen(command).read()
              print command + "\n" + result + "\n"

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-ip\":\"%s\", \"dst-ip\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".r", args.dstAddress, args.srcAddress, "0x800", ap2Port, ap1Port, controllerRestIp)
              result = os.popen(command).read()
              print command + "\n" + result + "\n"

            elif args.type == 'mac':

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-mac\":\"%s\", \"dst-mac\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".f", args.srcAddress, args.dstAddress, ap1Port, ap2Port, controllerRestIp)
              result = os.popen(command).read()
              print command + "\n" + result + "\n"

              command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"src-mac\":\"%s\", \"dst-mac\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".r", args.dstAddress, args.srcAddress, ap2Port, ap1Port, controllerRestIp)
              result = os.popen(command).read()
              print command + "\n" + result + "\n"

            command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".farp", "0x806", ap1Port, ap2Port, controllerRestIp)
            result = os.popen(command).read()
            print command + "\n" + result + "\n"

            command = "curl -s -d '{\"switch\": \"%s\", \"name\":\"%s\", \"ether-type\":\"%s\", \"cookie\":\"0\", \"priority\":\"32768\", \"ingress-port\":\"%s\",\"active\":\"true\", \"actions\":\"output=%s\"}' http://%s/wm/staticflowentrypusher/json" % (ap1Dpid, ap1Dpid+"."+args.circuitName+".rarp", "0x806", ap2Port, ap1Port, controllerRestIp)
            result = os.popen(command).read()
            print command + "\n" + result + "\n"

            # store created circuit attributes in local ./circuits.json
            datetime = time.asctime()
            circuitParams = {'name':args.circuitName, 'Dpid':ap1Dpid, 'inPort':ap1Port, 'outPort':ap2Port, 'datetime':datetime}
            str = json.dumps(circuitParams)
            circuitDb.write(str+"\n")

        # confirm successful circuit creation
        # using controller rest API

        command="curl -s http://%s/wm/core/switch/all/flow/json| python -mjson.tool" % (controllerRestIp)
        result = os.popen(command).read()
        print command + "\n" + result

elif args.action=='delete':

    circuitDb = open('./circuits.json','w')

    # removing previously created flow from switches
    # using StaticFlowPusher rest API
    # currently, circuitpusher records created circuits in local file ./circuits.db
    # with circuit name and list of switches

    circuitExists = False

    for line in lines:
        data = json.loads(line)
        if data['name']==(args.circuitName):
            circuitExists = True

            sw = data['Dpid']
            print data, sw

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".f", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".farp", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".r", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

            command = "curl -X DELETE -d '{\"name\":\"%s\", \"switch\":\"%s\"}' http://%s/wm/staticflowentrypusher/json" % (sw+"."+args.circuitName+".rarp", sw, controllerRestIp)
            result = os.popen(command).read()
            print command, result

        else:
            circuitDb.write(line)

    circuitDb.close()

    if not circuitExists:
        print "specified circuit does not exist"
        sys.exit(1)

<|endoftext|>