<commit_before>try:
    import ujson as json
except ImportError:
    import json
import urllib
import urllib2

from requests import Session

from . import config

# Search class
#_________________________________________________________________________________________
class Search(object):
    """This class represents an archive.org item search. You can use
    this class to search for archive.org items using the advanced
    search engine.

    Usage::

        >>> import internetarchive
        >>> search = internetarchive.Search('(uploader:jake@archive.org)')
        >>> for result in search.results:
        ...     print result['identifier']

    """

    # init()
    #_____________________________________________________________________________________
    def __init__(self, query, fields=['identifier'], params={}):
        self._base_url = 'http://archive.org/advancedsearch.php'
        self.query = query
        self.params = dict(dict(
                q = self.query,
                output = params.get('output', 'json'),
                rows = 100,
        ).items() + params.items())
        # Updata params dict with fields.
        for k, v in enumerate(fields):
            key = 'fl[{0}]'.format(k)
            self.params[key] = v
        self.encoded_params = urllib.urlencode(self.params)
        self.search_info = self._get_search_info()
        self.num_found = self.search_info['response']['numFound']

    # __repr__()
    #_____________________________________________________________________________________
    def __repr__(self):
        return ('Search(query={query!r}, '
                'num_found={num_found!r})'.format(**self.__dict__))

    # _get_search_info()
    #_____________________________________________________________________________________
    def _get_search_info(self):
        info_params = self.params.copy()
        info_params['rows'] = 0
        encoded_info_params = urllib.urlencode(info_params)
        f = urllib.urlopen(self._base_url, encoded_info_params)
        results = json.loads(f.read())
        del results['response']['docs']
        return results

    # _iter_results()
    #_____________________________________________________________________________________
    def results(self):
        """Generator for iterating over search results"""
        total_pages = ((self.num_found / self.params['rows']) + 2)
        for page in range(1, total_pages):
            self.params['page'] = page
            encoded_params = urllib.urlencode(self.params)
            f = urllib.urlopen(self._base_url, encoded_params)
            results = json.loads(f.read())
            for doc in results['response']['docs']:
                yield doc

#_________________________________________________________________________________________
class Catalog(object):
    """:todo: Document Catalog Class."""
    GREEN = 0
    BLUE = 1
    RED = 2
    BROWN = 9

    # init()
    #_____________________________________________________________________________________
    def __init__(self, identifier=None, params={}):
        params = {'justme': 1} if not params else params
        # Params required to retrieve JSONP from the IA catalog.
        params['json'] = 2
        params['output'] = 'json'
        params['callback'] = 'foo'

        self.session = Session()
        if identifier:
            self.url = 'http://archive.org/history/{id}'.format(id=identifier)
        else:
            self.url = 'http://archive.org/catalog.php'
        self.params = params
        self.session. cookies = config.get_cookiejar()
        self.tasks = self.get_tasks()

    # get_tasks()
    #_____________________________________________________________________________________
    def get_tasks(self):
        r = self.session.get(self.url, params=self.params)
        # Convert JSONP to JSON (then parse the JSON).
        json_str = r.content[(r.content.index("(") + 1):r.content.rindex(")")]
        return [CatalogTask(t) for t in json.loads(json_str)]

    # filter_tasks()
    #_____________________________________________________________________________________
    def filter_tasks(self, pred):
        return [t for t in self.tasks if pred(t)]

    # tasks_by_type()
    #_____________________________________________________________________________________
    def tasks_by_type(self, row_type):
        return self.filter_tasks(lambda t: t.row_type == row_type)

    # green_rows()
    #_____________________________________________________________________________________
    @property
    def green_rows(self):
        return self.tasks_by_type(self.GREEN)

    # blue_rows()
    #_____________________________________________________________________________________
    @property
    def blue_rows(self):
        return self.tasks_by_type(self.BLUE)

    # red_rows()
    #_____________________________________________________________________________________
    @property
    def red_rows(self):
        return self.tasks_by_type(self.RED)

    # brown_rows()
    #_____________________________________________________________________________________
    @property
    def brown_rows(self):
        return self.tasks_by_type(self.BROWN)

# CatalogTask class
#_________________________________________________________________________________________
class CatalogTask(object):
    """represents catalog task.
    """
    COLUMNS = ('identifier', 'server', 'command', 'time', 'submitter',
               'args', 'task_id', 'row_type')

    def __init__(self, columns):
        """:param columns: array of values, typically returned by catalog
        web service. see COLUMNS for the column name.
        """
        for a, v in map(None, self.COLUMNS, columns):
            if a: setattr(self, a, v)
        # special handling for 'args' - parse it into a dict if it is a string
        if isinstance(self.args, basestring):
            self.args = dict(x for x in urllib2.urlparse.parse_qsl(self.args))

    def __repr__(self):
        return ('CatalogTask(identifier={identifier},'
                ' task_id={task_id!r}, server={server!r},'
                ' command={command!r},'
                ' submitter={submitter!r},'
                ' row_type={row_type})'.format(**self.__dict__))

    def __getitem__(self, k):
        """dict-like access privided as backward compatibility."""
        if k in self.COLUMNS:
            return getattr(self, k, None)
        else:
            raise KeyError(k)

    def open_task_log(self):
        """return file-like reading task log."""
        if self.task_id is None:
            raise ValueError('task_id is None')
        url = 'http://catalogd.archive.org/log/{0}'.format(self.task_id)
        return urllib2.urlopen(url)
<commit_msg>set verbose cookie, so `ia catalog --verbose` outputs detailed information.<commit_after>try:
    import ujson as json
except ImportError:
    import json
import urllib
import urllib2

from requests import Session

from . import config

# Search class
#_________________________________________________________________________________________
class Search(object):
    """This class represents an archive.org item search. You can use
    this class to search for archive.org items using the advanced
    search engine.

    Usage::

        >>> import internetarchive
        >>> search = internetarchive.Search('(uploader:jake@archive.org)')
        >>> for result in search.results:
        ...     print result['identifier']

    """

    # init()
    #_____________________________________________________________________________________
    def __init__(self, query, fields=['identifier'], params={}):
        self._base_url = 'http://archive.org/advancedsearch.php'
        self.query = query
        self.params = dict(dict(
                q = self.query,
                output = params.get('output', 'json'),
                rows = 100,
        ).items() + params.items())
        # Updata params dict with fields.
        for k, v in enumerate(fields):
            key = 'fl[{0}]'.format(k)
            self.params[key] = v
        self.encoded_params = urllib.urlencode(self.params)
        self.search_info = self._get_search_info()
        self.num_found = self.search_info['response']['numFound']

    # __repr__()
    #_____________________________________________________________________________________
    def __repr__(self):
        return ('Search(query={query!r}, '
                'num_found={num_found!r})'.format(**self.__dict__))

    # _get_search_info()
    #_____________________________________________________________________________________
    def _get_search_info(self):
        info_params = self.params.copy()
        info_params['rows'] = 0
        encoded_info_params = urllib.urlencode(info_params)
        f = urllib.urlopen(self._base_url, encoded_info_params)
        results = json.loads(f.read())
        del results['response']['docs']
        return results

    # _iter_results()
    #_____________________________________________________________________________________
    def results(self):
        """Generator for iterating over search results"""
        total_pages = ((self.num_found / self.params['rows']) + 2)
        for page in range(1, total_pages):
            self.params['page'] = page
            encoded_params = urllib.urlencode(self.params)
            f = urllib.urlopen(self._base_url, encoded_params)
            results = json.loads(f.read())
            for doc in results['response']['docs']:
                yield doc

#_________________________________________________________________________________________
class Catalog(object):
    """:todo: Document Catalog Class."""
    GREEN = 0
    BLUE = 1
    RED = 2
    BROWN = 9

    # init()
    #_____________________________________________________________________________________
    def __init__(self, identifier=None, params={}, verbose=True):
        verbose = '1' if verbose else '0'
        params = {'justme': 1} if not params else params
        # Params required to retrieve JSONP from the IA catalog.
        params['json'] = 2
        params['output'] = 'json'
        params['callback'] = 'foo'

        self.session = Session()
        if identifier:
            self.url = 'http://archive.org/history/{id}'.format(id=identifier)
        else:
            self.url = 'http://archive.org/catalog.php'
        self.params = params
        self.session. cookies = config.get_cookiejar({'verbose': verbose})
        self.tasks = self.get_tasks()

    # get_tasks()
    #_____________________________________________________________________________________
    def get_tasks(self):
        r = self.session.get(self.url, params=self.params)
        # Convert JSONP to JSON (then parse the JSON).
        json_str = r.content[(r.content.index("(") + 1):r.content.rindex(")")]
        return [CatalogTask(t) for t in json.loads(json_str)]

    # filter_tasks()
    #_____________________________________________________________________________________
    def filter_tasks(self, pred):
        return [t for t in self.tasks if pred(t)]

    # tasks_by_type()
    #_____________________________________________________________________________________
    def tasks_by_type(self, row_type):
        return self.filter_tasks(lambda t: t.row_type == row_type)

    # green_rows()
    #_____________________________________________________________________________________
    @property
    def green_rows(self):
        return self.tasks_by_type(self.GREEN)

    # blue_rows()
    #_____________________________________________________________________________________
    @property
    def blue_rows(self):
        return self.tasks_by_type(self.BLUE)

    # red_rows()
    #_____________________________________________________________________________________
    @property
    def red_rows(self):
        return self.tasks_by_type(self.RED)

    # brown_rows()
    #_____________________________________________________________________________________
    @property
    def brown_rows(self):
        return self.tasks_by_type(self.BROWN)

# CatalogTask class
#_________________________________________________________________________________________
class CatalogTask(object):
    """represents catalog task.
    """
    COLUMNS = ('identifier', 'server', 'command', 'time', 'submitter',
               'args', 'task_id', 'row_type')

    def __init__(self, columns):
        """:param columns: array of values, typically returned by catalog
        web service. see COLUMNS for the column name.
        """
        for a, v in map(None, self.COLUMNS, columns):
            if a: setattr(self, a, v)
        # special handling for 'args' - parse it into a dict if it is a string
        if isinstance(self.args, basestring):
            self.args = dict(x for x in urllib2.urlparse.parse_qsl(self.args))

    def __repr__(self):
        return ('CatalogTask(identifier={identifier},'
                ' task_id={task_id!r}, server={server!r},'
                ' command={command!r},'
                ' submitter={submitter!r},'
                ' row_type={row_type})'.format(**self.__dict__))

    def __getitem__(self, k):
        """dict-like access privided as backward compatibility."""
        if k in self.COLUMNS:
            return getattr(self, k, None)
        else:
            raise KeyError(k)

    def open_task_log(self):
        """return file-like reading task log."""
        if self.task_id is None:
            raise ValueError('task_id is None')
        url = 'http://catalogd.archive.org/log/{0}'.format(self.task_id)
        return urllib2.urlopen(url)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse, sys
from collections import OrderedDict
from utilities import *

def parse_args(args):
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(
        description='Converts hub info metadata from csv to json.')
    parser.add_argument('input') # Absence of dash denotes mandatory argument
    return parser.parse_args(args)

def main(test_args=None):
    """main"""
    args = parse_args(sys.argv[1:] if test_args is None else test_args)

    csv_data = read_csv(args.input)
    new_data = OrderedDict({})
    for i in range(int(len(csv_data)/2)):
        row = csv_data[int(2*i)]
        chains = csv_data[int(2*i+1)]
        component_data = OrderedDict({})
        for i in range(int(len(chains)/2)):
            component_data[chains[int(2*i)]] = \
                { 'single_name': chains[int(2*i+1)],
                'c_free': row[4]=='C_free',
                'n_free': row[3]=='N_free' }
        new_data[row[0].replace('.pdb', '')] = \
            OrderedDict({ 'oligomer_type': row[1],
                'symmetric': row[2] == 'symmetric',
                'component_data': component_data})
    json.dump(new_data,
        open(args.input.replace('.csv', '') + '.json', 'w'),
        separators=(',',':'),
        ensure_ascii=False,
        indent=4)

if __name__ == '__main__':
    safe_exec(main)<commit_msg>added more flexibility in creating hub_info.json<commit_after>#!/usr/bin/env python3

import argparse, sys
from collections import OrderedDict
from utilities import *

def parse_args(args):
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(
        description='Converts hub info metadata from csv to json.')
    parser.add_argument('input') # Absence of dash denotes mandatory argument
    return parser.parse_args(args)

def main(test_args=None):
    """main"""
    args = parse_args(sys.argv[1:] if test_args is None else test_args)

    csv_data = read_csv(args.input)
    new_data = OrderedDict({})
    for i in range(int(len(csv_data)/2)): # for half the length of the rows of the file do
        row = csv_data[int(2*i)]
        chains = csv_data[int(2*i+1)]
        component_data = OrderedDict({})
        for i in range(int(len(chains)/4)):
            component_data[chains[int(4*i)]] = \
                { 'single_name': chains[int(4*i+1)],
                'c_free': chains[int(4*i+3)] =='C_free',
                'n_free': chains[int(4*i+2)] =='N_free' }
        new_data[row[0].replace('.pdb', '')] = \
            OrderedDict({ 'oligomer_type': row[1],
            OrderedDict({ 'oligomer_type': row[1],
                'symmetric': row[2] == 'symmetric',
                'component_data': component_data})
    json.dump(new_data,
        open(args.input.replace('.csv', '') + '.json', 'w'),
        separators=(',',':'),
        ensure_ascii=False,
        indent=4)

if __name__ == '__main__':
    safe_exec(main)<|endoftext|>


<commit_before>from django import forms

import csv
from io import StringIO

class BatchMoonScanForm(forms.Form):
    data = forms.CharField(
        widget=forms.Textarea(attrs={'class':'form-control monospace'}),
    )

    def clean(self):
        cleaned_data = super(BatchMoonScanForm, self).clean()
        raw = StringIO(cleaned_data['data'])
        reader = csv.reader(raw, delimiter='\t')

        next(reader)

        res = []

        for x in reader:
            print(x)
            if len(x) == 1:
                assert(len(x[0]) > 0)

                current_moon = 0
                current_scan = {}
                res.append(current_scan)
            else:
                assert(len(x[0]) == 0)

                moon_id = int(x[6])
                ore_id = int(x[3])
                percentage = int(round(100 * float(x[2])))

                if current_moon == 0:
                    current_moon = moon_id
                else:
                    assert(moon_id == current_moon)

                assert(ore_id not in current_scan)

                current_scan[ore_id] = percentage

        print(res)
        cleaned_data['data'] = res
<commit_msg>Improve batch form return data structure.<commit_after>from django import forms

import csv
from io import StringIO

class BatchMoonScanForm(forms.Form):
    data = forms.CharField(
        widget=forms.Textarea(attrs={'class':'form-control monospace'}),
    )

    def clean(self):
        cleaned_data = super(BatchMoonScanForm, self).clean()
        raw = StringIO(cleaned_data['data'])
        reader = csv.reader(raw, delimiter='\t')

        next(reader)

        res = {}
        current_moon = 0
        percentage_sum = 0
        current_scan = {}

        for x in reader:
            print(x)
            if len(x) == 1:
                if len(x[0]) == 0:
                    raise forms.ValidationError('Invalid input format.')

                if current_moon != 0 and percentage_sum != 100:
                    raise forms.ValidationError('Sum of percentages must be 100.')

                if len(current_scan) > 0 and current_moon != 0:
                    res[current_moon] = current_scan

                current_moon = 0
                percentage_sum = 0
                current_scan = {}
            else:
                if len(x[0]) != 0:
                    raise forms.ValidationError('Invalid input format.')

                moon_id = int(x[6])
                ore_id = int(x[3])
                percentage = int(round(100 * float(x[2])))

                percentage_sum += percentage

                if current_moon == 0:
                    current_moon = moon_id
                elif moon_id != current_moon:
                    raise forms.ValidationError('Unexpected moon ID.')

                if ore_id in current_scan:
                    raise forms.ValidationError('Unexpected moon ID.')

                current_scan[ore_id] = percentage

        print(res)
        cleaned_data['data'] = res
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" This module contains the classes required to manage a hosts file """
import sys
import exception
from utils import is_ipv4, is_ipv6, valid_hostnames, is_readable, is_writeable
import urllib2

class HostsEntry(object):
    """ An entry in a hosts file. """
    __slots__ = ['entry_type', 'address', 'comment', 'names']

    def __init__(self, entry_type=None, address=None, comment=None, names=None):
        if not entry_type or entry_type not in ('ipv4',
                                                'ipv6',
                                                'comment',
                                                'blank'):
            raise Exception('entry_type invalid or not specified')

        if entry_type == 'comment' and not comment:
            raise Exception('entry_type comment supplied without value.')

        if entry_type == 'ipv4':
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')
            if not is_ipv4(address):
                raise exception.InvalidIPv4Address()

        if entry_type == 'ipv6':
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')
            if not is_ipv6(address):
                raise exception.InvalidIPv6Address()

        self.entry_type = entry_type
        self.address = address
        self.comment = comment
        self.names = names

    @staticmethod
    def get_entry_type(hosts_entry=None):
        """
        Return the type of entry for the line of hosts file passed
        :param hosts_entry: A line from the hosts file
        :return: comment | blank | ipv4 | ipv6
        """
        if hosts_entry and isinstance(hosts_entry, str):
            entry = hosts_entry.strip()
            if not entry or not entry[0] or entry[0] == "\n":
                return 'blank'
            if entry[0] == "#":
                return 'comment'
            entry_chunks = entry.split()
            if is_ipv6(entry_chunks[0]):
                return 'ipv6'
            if is_ipv4(entry_chunks[0]):
                return 'ipv4'

    @staticmethod
    def str_to_hostentry(entry):
        if isinstance(entry, str):
            line_parts = entry.strip().split()
            if is_ipv4(line_parts[0]):
                if valid_hostnames(line_parts[1:]):
                    return HostsEntry(entry_type='ipv4', address=line_parts[0], names=line_parts[1:])
            elif is_ipv6(line_parts[0]):
                if valid_hostnames(line_parts[1:]):
                    return HostsEntry(entry_type='ipv6', address=line_parts[0], names=line_parts[1:])
            else:
                return False

class Hosts(object):
    """ A hosts file. """
    __slots__ = ['entries', 'hosts_path']

    def __init__(self, path=None):
        """
        Returns a list of all entries in the hosts file.
        Each entry is represented in the form of a dict.
        """
        self.entries = []
        if path:
            self.hosts_path = path
        else:
            self.hosts_path = self.determine_hosts_path()
        self.populate_entries()

    @staticmethod
    def determine_hosts_path(platform=None):
        """
        Return the hosts file path based on the supplied
        or detected platform.
        :param platform: override platform detection
        :return: path of hosts file
        """
        if not platform:
            platform = sys.platform
        if platform.startswith('win'):
            return r'c:\windows\system32\drivers\etc\hosts'
        else:
            return '/etc/hosts'

    def dedupe_entries(self):
        import hashlib
        existing_hashes = []
        deduped = []
        for entry in self.entries:
            if entry.entry_type in ('ipv4', 'ipv6'):
                entry_hash = hashlib.sha224("{0}{1}{2}".format(entry.entry_type,
                                                               entry.address,
                                                               entry.names
                                                               )
                                            ).hexdigest()
                if entry_hash not in existing_hashes:
                    deduped.append(entry)
                    existing_hashes.append(entry_hash)
        self.entries = deduped

    def write(self):
        """
        Write the list of host entries back to the hosts file.
        """
        written_count = 0
        comments_written = 0
        blanks_written = 0
        ipv4_entries_written = 0
        ipv6_entries_written = 0
        if is_writeable(self.hosts_path):
            with open(self.hosts_path, 'w') as hosts_file:
                for written_count, line in enumerate(self.entries):
                    if line.entry_type == 'comment':
                        hosts_file.write(line.comment)
                        comments_written += 1
                    if line.entry_type == 'blank':
                        hosts_file.write("\n")
                        blanks_written += 1
                    if line.entry_type == 'ipv4':
                        hosts_file.write(
                            "{0}\t{1}\n".format(
                                line.address,
                                ' '.join(line.names),
                                )
                            )
                        ipv4_entries_written += 1
                    if line.entry_type == 'ipv6':
                        hosts_file.write(
                            "{0}\t{1}\n".format(
                                line.address,
                                ' '.join(line.names),))
                        ipv6_entries_written += 1
                return {'total_written': written_count+1,
                        'comments_written': comments_written,
                        'blanks_written': blanks_written,
                        'ipv4_entries_written': ipv4_entries_written,
                        'ipv6_entries_written': ipv6_entries_written}

    @staticmethod
    def get_hosts_by_url(url=None):
        response = urllib2.urlopen(url)
        return response.read()

    def import_url(self, url=None):
        file_contents = self.get_hosts_by_url(url=url)
        file_contents = file_contents.rstrip().replace('^M', '\n')
        file_contents = file_contents.rstrip().replace('\r\n', '\n')
        lines = file_contents.split('\n')
        skipped = 0
        import_entries = []
        for line in lines:
            stripped_entry = line.strip()
            if (not stripped_entry) or (stripped_entry.startswith('#')):
                skipped += 1
            else:
                import_entry = HostsEntry.str_to_hostentry(line)
                import_entries.append(import_entry)
        add_result = self.add(entries=import_entries)
        self.dedupe_entries()
        write_result = self.write()
        return {'result': 'success',
                'skipped': skipped,
                'add_result': add_result,
                'write_result': write_result}

    def import_file(self, import_file_path=None):
        skipped = 0
        if is_readable(import_file_path):
            import_entries = []
            with open(import_file_path, 'r') as infile:
                for entries_count, line in enumerate(infile):
                    stripped_entry = line.strip()
                    if (not stripped_entry) or (stripped_entry.startswith('#')):
                        skipped += 1
                    else:
                        import_entry = HostsEntry.str_to_hostentry(line)
                        import_entries.append(import_entry)
                add_result = self.add(entries=import_entries)
                self.dedupe_entries()
                write_result = self.write()
                return {'result': 'success',
                        'skipped': skipped,
                        'add_result': add_result,
                        'write_result': write_result}
        else:
            return {'result': 'failed',
                    'message': 'Cannot read: file {0}.'.format(import_file_path)}

    def exists(self, entry):
        address_matches = 0
        name_matches = 0
        for existing_entry in self.entries:
            if entry.entry_type == existing_entry.entry_type:
                if entry.address == existing_entry.address:
                    address_matches += 1
                if set(existing_entry.names).intersection(set(entry.names)):
                    name_matches += 1
        return {'address_matches': address_matches,
                'name_matches': name_matches}

    def remove_matching(self, entry=None, address=None, names=None):
        names_list = []
        if names:
            names = names.replace(',', ' ')
            names_list = names.split()
        removed_count = 0
        if not any((entry, address, names)):
            return removed_count
        entries_to_remove = []
        if entry:
            for existing_entry in self.entries:
                if entry.entry_type == existing_entry.entry_type:
                    if entry.address == existing_entry.address:
                        self.entries.remove(existing_entry)
                        removed_count += 1
                        continue
                    for name in entry.names:
                        if name in existing_entry.names:
                            entries_to_remove.append(existing_entry)
                            removed_count += 1
                            continue
        elif address or names_list:
            for existing_entry in self.entries:
                if existing_entry.entry_type in ('ipv4', 'ipv6'):
                    if address == existing_entry.address or set(names_list).intersection(set(existing_entry.names)):
                        self.entries.remove(existing_entry)
                        removed_count += 1

        purged_list = [x for x in self.entries if x not in entries_to_remove]
        self.entries = purged_list
        return removed_count

    def add(self, entries=None, force=None):
        """
        Adds to the instance list of entries.
        :param entries: A list of instances of HostsEntry
        :return: The number of successes and failures
        """
        ipv4_count = 0
        ipv6_count = 0
        invalid_count = 0
        duplicate_count = 0
        replaced_count = 0

        for count, item in enumerate(entries):
            if not isinstance(item, HostsEntry):
                invalid_count += 1
                continue
            if item.entry_type == 'ipv4':
                exists = self.exists(item)
                if (exists.get('name_matches')) and (item.address in ('0.0.0.0', '127.0.0.1')):
                    duplicate_count += 1
                    continue
                elif item.address in ('0.0.0.0', '127.0.0.1'):
                    ipv4_count += 1
                elif exists.get('address_matches') or exists.get('name_matches'):
                    if force:
                        replaced_count += self.remove_matching(item)
                        ipv4_count += 1
                    else:
                        duplicate_count += 1
                        continue
                else:
                    ipv4_count += 1
            elif item.entry_type == 'ipv6':
                exists_res = self.exists(item)
                if exists_res.get('address_matches') or exists_res.get('name_matches'):
                    if force:
                        replaced_count += self.remove_matching(item)
                        ipv6_count += 1
                    else:
                        duplicate_count += 1
                        continue
            self.entries.append(item)
        return {'ipv4_count': ipv4_count,
                'ipv6_count': ipv6_count,
                'invalid_count': invalid_count,
                'duplicate_count': duplicate_count,
                'replaced_count': replaced_count}

    def populate_entries(self):
        """
        Read all hosts file entries from the hosts file specified
        and store them as HostEntry's in an instance of Hosts.
        """
        try:
            with open(self.hosts_path, 'r') as hosts_file:
                hosts_entries = [line for line in hosts_file]
                for hosts_entry in hosts_entries:
                    entry_type = HostsEntry.get_entry_type(hosts_entry)
                    if entry_type == "comment":
                        self.entries.append(HostsEntry(entry_type="comment",
                                                       comment=hosts_entry))
                    elif entry_type == "blank":
                        self.entries.append(HostsEntry(entry_type="blank"))
                    elif entry_type == "ipv4" or entry_type == "ipv6":
                        chunked_entry = hosts_entry.split()
                        stripped_name_list = []
                        for name in chunked_entry[1:]:
                            stripped_name_list.append(name.strip())

                        self.entries.append(
                            HostsEntry(
                                entry_type=entry_type,
                                address=chunked_entry[0].strip(),
                                names=stripped_name_list))
        except IOError:
                return {'result': 'failed',
                        'message': 'Cannot read: {0}.'.format(self.hosts_path)}
<commit_msg>preserve existing comments and blanks.<commit_after># -*- coding: utf-8 -*-
""" This module contains the classes required to manage a hosts file """
import sys
import exception
from utils import is_ipv4, is_ipv6, valid_hostnames, is_readable, is_writeable
import urllib2

class HostsEntry(object):
    """ An entry in a hosts file. """
    __slots__ = ['entry_type', 'address', 'comment', 'names']

    def __init__(self, entry_type=None, address=None, comment=None, names=None):
        if not entry_type or entry_type not in ('ipv4',
                                                'ipv6',
                                                'comment',
                                                'blank'):
            raise Exception('entry_type invalid or not specified')

        if entry_type == 'comment' and not comment:
            raise Exception('entry_type comment supplied without value.')

        if entry_type == 'ipv4':
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')
            if not is_ipv4(address):
                raise exception.InvalidIPv4Address()

        if entry_type == 'ipv6':
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')
            if not is_ipv6(address):
                raise exception.InvalidIPv6Address()

        self.entry_type = entry_type
        self.address = address
        self.comment = comment
        self.names = names

    @staticmethod
    def get_entry_type(hosts_entry=None):
        """
        Return the type of entry for the line of hosts file passed
        :param hosts_entry: A line from the hosts file
        :return: comment | blank | ipv4 | ipv6
        """
        if hosts_entry and isinstance(hosts_entry, str):
            entry = hosts_entry.strip()
            if not entry or not entry[0] or entry[0] == "\n":
                return 'blank'
            if entry[0] == "#":
                return 'comment'
            entry_chunks = entry.split()
            if is_ipv6(entry_chunks[0]):
                return 'ipv6'
            if is_ipv4(entry_chunks[0]):
                return 'ipv4'

    @staticmethod
    def str_to_hostentry(entry):
        if isinstance(entry, str):
            line_parts = entry.strip().split()
            if is_ipv4(line_parts[0]):
                if valid_hostnames(line_parts[1:]):
                    return HostsEntry(entry_type='ipv4', address=line_parts[0], names=line_parts[1:])
            elif is_ipv6(line_parts[0]):
                if valid_hostnames(line_parts[1:]):
                    return HostsEntry(entry_type='ipv6', address=line_parts[0], names=line_parts[1:])
            else:
                return False

class Hosts(object):
    """ A hosts file. """
    __slots__ = ['entries', 'hosts_path']

    def __init__(self, path=None):
        """
        Returns a list of all entries in the hosts file.
        Each entry is represented in the form of a dict.
        """
        self.entries = []
        if path:
            self.hosts_path = path
        else:
            self.hosts_path = self.determine_hosts_path()
        self.populate_entries()

    @staticmethod
    def determine_hosts_path(platform=None):
        """
        Return the hosts file path based on the supplied
        or detected platform.
        :param platform: override platform detection
        :return: path of hosts file
        """
        if not platform:
            platform = sys.platform
        if platform.startswith('win'):
            return r'c:\windows\system32\drivers\etc\hosts'
        else:
            return '/etc/hosts'

    def dedupe_entries(self):
        import hashlib
        existing_hashes = []
        deduped = []
        for entry in self.entries:
            if entry.entry_type in ('ipv4', 'ipv6'):
                entry_hash = hashlib.sha224("{0}{1}{2}".format(entry.entry_type,
                                                               entry.address,
                                                               entry.names
                                                               )
                                            ).hexdigest()
                if entry_hash not in existing_hashes:
                    deduped.append(entry)
                    existing_hashes.append(entry_hash)
            if entry.entry_type in ('comment', 'blank'):
                    deduped.append(entry)
        self.entries = deduped

    def write(self):
        """
        Write the list of host entries back to the hosts file.
        """
        written_count = 0
        comments_written = 0
        blanks_written = 0
        ipv4_entries_written = 0
        ipv6_entries_written = 0
        if is_writeable(self.hosts_path):
            with open(self.hosts_path, 'w') as hosts_file:
                for written_count, line in enumerate(self.entries):
                    if line.entry_type == 'comment':
                        hosts_file.write(line.comment)
                        comments_written += 1
                    if line.entry_type == 'blank':
                        hosts_file.write("\n")
                        blanks_written += 1
                    if line.entry_type == 'ipv4':
                        hosts_file.write(
                            "{0}\t{1}\n".format(
                                line.address,
                                ' '.join(line.names),
                                )
                            )
                        ipv4_entries_written += 1
                    if line.entry_type == 'ipv6':
                        hosts_file.write(
                            "{0}\t{1}\n".format(
                                line.address,
                                ' '.join(line.names),))
                        ipv6_entries_written += 1
                return {'total_written': written_count+1,
                        'comments_written': comments_written,
                        'blanks_written': blanks_written,
                        'ipv4_entries_written': ipv4_entries_written,
                        'ipv6_entries_written': ipv6_entries_written}

    @staticmethod
    def get_hosts_by_url(url=None):
        response = urllib2.urlopen(url)
        return response.read()

    def import_url(self, url=None):
        file_contents = self.get_hosts_by_url(url=url)
        file_contents = file_contents.rstrip().replace('^M', '\n')
        file_contents = file_contents.rstrip().replace('\r\n', '\n')
        lines = file_contents.split('\n')
        skipped = 0
        import_entries = []
        for line in lines:
            stripped_entry = line.strip()
            if (not stripped_entry) or (stripped_entry.startswith('#')):
                skipped += 1
            else:
                import_entry = HostsEntry.str_to_hostentry(line)
                import_entries.append(import_entry)
        add_result = self.add(entries=import_entries)
        self.dedupe_entries()
        write_result = self.write()
        return {'result': 'success',
                'skipped': skipped,
                'add_result': add_result,
                'write_result': write_result}

    def import_file(self, import_file_path=None):
        skipped = 0
        if is_readable(import_file_path):
            import_entries = []
            with open(import_file_path, 'r') as infile:
                for entries_count, line in enumerate(infile):
                    stripped_entry = line.strip()
                    if (not stripped_entry) or (stripped_entry.startswith('#')):
                        skipped += 1
                    else:
                        import_entry = HostsEntry.str_to_hostentry(line)
                        import_entries.append(import_entry)
                add_result = self.add(entries=import_entries)
                self.dedupe_entries()
                write_result = self.write()
                return {'result': 'success',
                        'skipped': skipped,
                        'add_result': add_result,
                        'write_result': write_result}
        else:
            return {'result': 'failed',
                    'message': 'Cannot read: file {0}.'.format(import_file_path)}

    def exists(self, entry):
        address_matches = 0
        name_matches = 0
        for existing_entry in self.entries:
            if entry.entry_type == existing_entry.entry_type:
                if entry.address == existing_entry.address:
                    address_matches += 1
                if set(existing_entry.names).intersection(set(entry.names)):
                    name_matches += 1
        return {'address_matches': address_matches,
                'name_matches': name_matches}

    def remove_matching(self, entry=None, address=None, names=None):
        names_list = []
        if names:
            names = names.replace(',', ' ')
            names_list = names.split()
        removed_count = 0
        if not any((entry, address, names)):
            return removed_count
        entries_to_remove = []
        if entry:
            for existing_entry in self.entries:
                if entry.entry_type == existing_entry.entry_type:
                    if entry.address == existing_entry.address:
                        self.entries.remove(existing_entry)
                        removed_count += 1
                        continue
                    for name in entry.names:
                        if name in existing_entry.names:
                            entries_to_remove.append(existing_entry)
                            removed_count += 1
                            continue
        elif address or names_list:
            for existing_entry in self.entries:
                if existing_entry.entry_type in ('ipv4', 'ipv6'):
                    if address == existing_entry.address or set(names_list).intersection(set(existing_entry.names)):
                        self.entries.remove(existing_entry)
                        removed_count += 1

        purged_list = [x for x in self.entries if x not in entries_to_remove]
        self.entries = purged_list
        return removed_count

    def add(self, entries=None, force=None):
        """
        Adds to the instance list of entries.
        :param entries: A list of instances of HostsEntry
        :return: The number of successes and failures
        """
        ipv4_count = 0
        ipv6_count = 0
        invalid_count = 0
        duplicate_count = 0
        replaced_count = 0

        for count, item in enumerate(entries):
            if not isinstance(item, HostsEntry):
                invalid_count += 1
                continue
            if item.entry_type == 'ipv4':
                exists = self.exists(item)
                if (exists.get('name_matches')) and (item.address in ('0.0.0.0', '127.0.0.1')):
                    duplicate_count += 1
                    continue
                elif item.address in ('0.0.0.0', '127.0.0.1'):
                    ipv4_count += 1
                elif exists.get('address_matches') or exists.get('name_matches'):
                    if force:
                        replaced_count += self.remove_matching(item)
                        ipv4_count += 1
                    else:
                        duplicate_count += 1
                        continue
                else:
                    ipv4_count += 1
            elif item.entry_type == 'ipv6':
                exists_res = self.exists(item)
                if exists_res.get('address_matches') or exists_res.get('name_matches'):
                    if force:
                        replaced_count += self.remove_matching(item)
                        ipv6_count += 1
                    else:
                        duplicate_count += 1
                        continue
            self.entries.append(item)
        return {'ipv4_count': ipv4_count,
                'ipv6_count': ipv6_count,
                'invalid_count': invalid_count,
                'duplicate_count': duplicate_count,
                'replaced_count': replaced_count}

    def populate_entries(self):
        """
        Read all hosts file entries from the hosts file specified
        and store them as HostEntry's in an instance of Hosts.
        """
        try:
            with open(self.hosts_path, 'r') as hosts_file:
                hosts_entries = [line for line in hosts_file]
                for hosts_entry in hosts_entries:
                    entry_type = HostsEntry.get_entry_type(hosts_entry)
                    if entry_type == "comment":
                        self.entries.append(HostsEntry(entry_type="comment",
                                                       comment=hosts_entry))
                    elif entry_type == "blank":
                        self.entries.append(HostsEntry(entry_type="blank"))
                    elif entry_type == "ipv4" or entry_type == "ipv6":
                        chunked_entry = hosts_entry.split()
                        stripped_name_list = []
                        for name in chunked_entry[1:]:
                            stripped_name_list.append(name.strip())

                        self.entries.append(
                            HostsEntry(
                                entry_type=entry_type,
                                address=chunked_entry[0].strip(),
                                names=stripped_name_list))
        except IOError:
                return {'result': 'failed',
                        'message': 'Cannot read: {0}.'.format(self.hosts_path)}
<|endoftext|>


<commit_before># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Controller framework
"""

import logging
import struct
import re
import sqlite3
import os
import sys
from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import CONFIG_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_0
from ryu.ofproto import ether
from ryu.lib.mac import haddr_to_bin
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.lib.packet import ipv4
from ryu.lib.packet import arp
from ryu.lib.packet import tcp
from ryu.controller import dpset
from ryu.lib.packet import stream_parser
from ryu.lib.packet import packet_base
from netaddr import *
from utils import *
from ryu.lib.mac import haddr_to_bin
#from PropFair import *

'''
This file is edited from Ryu example which is located at  ryu/ryu/app/simple_switch.py.
According to its licecse(please don't trust my reading and read it), we can modify and use it as long as we keep the old license and state we've change the code. --Joe
'''

FLOW_HARD_TIMEOUT = 30
FLOW_IDLE_TIMEOUT = 10
CONTROLLER_IP = "10.10.4.1"
CONTROLLER_MAC = "00:00:00:00:00:01"

class SimpleSwitch(app_manager.RyuApp):
    global printComments

    OFP_VERSIONS = [ofproto_v1_0.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.ip_mac_port = {}  #collection of all interfaces connected to OVS
	printComments = 1
	self.configuration = {}  #table for the configuration: DEFAULT/IP_addess;{PARAM*WEIGHT};TIMEOUT
	self.number_of_servers = {}
	self.servers = {}
	#read the configuration file
	__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
	with open(os.path.join(__location__, 'config')) as f:
           for line in f:
             if line.startswith( '#' ):
	       continue
	     split_line = re.split(';',line[:-1])
             server = split_line[0]
             parameters_weights = re.split(',',split_line[1])
             parameter_list = []
             for pw in parameters_weights:
               parameter = re.split('\*',pw)
               if len(parameter)==1:
                  #in case if we don't specify the weight
                  parameter.append(100/len(parameters_weights))
               parameter_list.append(tuple(parameter))
	     timeout = split_line[2]
	     self.configuration.setdefault(server, {})
             self.configuration[server] = (parameter_list, timeout)
	print self.configuration

        if "DEFAULT" not in self.configuration:
           print "WRONG CONFIGURATION"
	   exit()

        print "Configuration validated"

    def add_flow(self, datapath, match, act, priority=0, idle_timeout=0, flags=0, cookie=0):
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser
        mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, actions=act, flags=flags, idle_timeout=idle_timeout, cookie=cookie)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        print "Packet in"
        #parse the message from the event
        msg = ev.msg
	in_port = msg.in_port
        #switch id
        datapath = msg.datapath
        dpid = datapath.id
        ofproto = datapath.ofproto
        #set default key in a dictionary
        self.ip_mac_port.setdefault(dpid, {})

        #read the packet
        pkt = packet.Packet(msg.data)
	eth = pkt.get_protocol(ethernet.ethernet)
        dst = eth.dst
        src = eth.src

        arp_pkt = pkt.get_protocol(arp.arp)
        if arp_pkt:
           self.logger.info("arp %s", arp_pkt)
           src_ip = arp_pkt.src_ip
           dst_ip = arp_pkt.dst_ip

           #save ip to mac and port correspondance
           self.ip_mac_port[dpid][src_ip] = (src, in_port)
           print self.ip_mac_port[dpid][src_ip]

           self.send_arp_reply(dpid, datapath, eth, arp_pkt, src_ip, dst_ip, in_port, msg)

        ipv4_pkt = pkt.get_protocol(ipv4.ipv4)
        if ipv4_pkt:
           self.logger.info("ipv4_packet %s", ipv4_pkt)

        udp_segment = pkt.get_protocol(udp.udp)
        if udp_segment and ipv4_pkt.dst == CONTROLLER_IP:
           self.logger.info("udp %s", udp_segment)
           print "Got UDP packet"
           udp_pointer = len(msg.data) - udp_segment.total_length + 8
           #udp_parsed = stream_parser.StreamParser().parse(msg.data)
           message = msg.data[udp_pointer:]
           if udp_segment.dst_port == 7777 and message == "Hello":
             print "Server initialization"
             #send UDP reply with the list of parameters and timeout
    	     if ipv4_pkt.src in self.configuration:
                conf_src = ipv4_pkt.src
             else:
                conf_src = "DEFAULT"

            # if ipv4_pkt.src not in self.servers[dpid][(lambda x: x)(self.number_of_servers[dpid])][0]:
            # if ipv4_pkt.src not in self.servers[dpid][(lambda x: x)(self.number_of_servers[dpid])][0]:
	     serverID = findIPInServerList(self.servers[dpid], ipv4_pkt.src)
	     if serverID==-1:
               print "Add new server"
               serverID = self.number_of_servers[dpid] = self.number_of_servers[dpid] + 1
               self.servers[dpid][serverID] = [ipv4_pkt.src,[]]
               self.servers[dpid][serverID][1]=[0.0]*len(self.configuration[conf_src][0])

             message = ','.join(p[0] for p in self.configuration[conf_src][0]) + ";" + self.configuration[conf_src][1] + ";" + str(serverID)
	     self.send_udp_reply(dpid, datapath, eth, ipv4_pkt, udp_segment, in_port, message)

             print self.servers

           elif udp_segment.dst_port == 7778:
             print message
             recieved_data = re.split(';', message)
	     serverID = int(recieved_data[1])
             if serverID not in self.servers[dpid]:
                print "Server not registered"
             else:
	       values = re.split(',', recieved_data[0])
               i = 0
               for value in values:
                 self.servers[dpid][serverID][1][i] = float(value)
                 i += 1
               print self.servers

    def send_udp_reply(self, dpid, datapath, eth, ipv4_pkt, udp_segment, out_port, message):
	ofproto = datapath.ofproto
        e = ethernet.ethernet(dst=eth.src, src=CONTROLLER_MAC, ethertype=ether.ETH_TYPE_IP)
        i = ipv4.ipv4(dst=ipv4_pkt.src, src=ipv4_pkt.dst, proto=17, total_length=0)
        u = udp.udp(src_port=udp_segment.dst_port, dst_port=udp_segment.src_port, total_length=0, csum=0)
        udp_h = u.serialize(message, i)
        ip_h = i.serialize(udp_h + message, e)
        eth_h = e.serialize(ip_h + udp_h + message, None)

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port)]
        data = eth_h + ip_h + udp_h + message

	out = datapath.ofproto_parser.OFPPacketOut(datapath=datapath, buffer_id=0xffffffff,
            in_port=datapath.ofproto.OFPP_CONTROLLER,
            actions=actions, data=data)
	datapath.send_msg(out)

    def send_arp_reply(self, dpid, datapath, eth, arp_pkt, src_ip, dst_ip, in_port, msg):
	ofproto = datapath.ofproto
        out_port = None

	if dst_ip == CONTROLLER_IP:
           print "Controller ARP request"
           e = ethernet.ethernet(dst=eth.src, src=CONTROLLER_MAC, ethertype=ether.ETH_TYPE_ARP)
           a = arp.arp(hwtype=1, proto=0x0800, hlen=6, plen=4, opcode=2, src_mac=CONTROLLER_MAC,
               src_ip=CONTROLLER_IP, dst_mac=eth.src, dst_ip=src_ip)

        elif dst_ip in self.ip_mac_port[dpid]:
           out_port = self.ip_mac_port[dpid][dst_ip][1]
        else:
           out_port = ofproto.OFPP_FLOOD

        if (out_port != None):
           actions = [datapath.ofproto_parser.OFPActionOutput(out_port)]
           data = msg.data

	else:
	   p = packet.Packet()
           p.add_protocol(e)
           p.add_protocol(a)
           p.serialize()
           actions = [datapath.ofproto_parser.OFPActionOutput(in_port)]
           data = p.data

	out = datapath.ofproto_parser.OFPPacketOut(datapath=datapath, buffer_id=0xffffffff,
            in_port=datapath.ofproto.OFPP_CONTROLLER,
            actions=actions, data=data)
	datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPFlowRemoved, MAIN_DISPATCHER)
    def flow_removal_handler(self, ev):
        msg = ev.msg

    def remove_table_flows(self, datapath, table_id, match):
        """Create OFP flow mod message to remove flows from table."""
        ofproto = datapath.ofproto
        flow_mod = datapath.ofproto_parser.OFPFlowMod(datapath=datapath, match=match, command=ofproto.OFPFC_DELETE,
                             cookie=0, idle_timeout=0,out_port=65535, buffer_id=4294967295, flags=0, hard_timeout=0,priority=0, actions=[])
        return flow_mod

    @set_ev_cls(dpset.EventDP, dpset.DPSET_EV_DISPATCHER)
    def _event_switch_enter_handler(self, ev):
       #definitions
       actions = []
       dl_type_ipv4 = 0x0800
       dl_type_arp = 0x0806
       dp = ev.dp
       dpid = dp.id

       ofproto = dp.ofproto
       parser = dp.ofproto_parser
       empty_match = parser.OFPMatch()
       instructions = []

       self.number_of_servers.setdefault(dpid, {})
       self.servers.setdefault(dpid, {})
       self.number_of_servers[dpid]=0
       self.servers[dpid][0]=('0')

       #empty the flow table
       flow_mod = self.remove_table_flows(dp, 0,empty_match)
       dp.send_msg(flow_mod)

       #process arp packets normally
       actions = [parser.OFPActionOutput(ofproto.OFPP_NORMAL)]
       match = parser.OFPMatch(dl_type = dl_type_arp)
#       self.add_flow(dp, match, actions, priority=100)

       #add miss flow
       match = parser.OFPMatch ()
       actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER)]
       self.add_flow(dp, match, actions, priority=1)

<commit_msg>Adding the TCP proccesing<commit_after># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Controller framework
"""

import logging
import struct
import re
import sqlite3
import os
import sys
from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import CONFIG_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_0
from ryu.ofproto import ether
from ryu.lib.mac import haddr_to_bin
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.lib.packet import ipv4
from ryu.lib.packet import arp
from ryu.lib.packet import tcp
from ryu.controller import dpset
from ryu.lib.packet import stream_parser
from ryu.lib.packet import packet_base
from netaddr import *
from utils import *
from ryu.lib.mac import haddr_to_bin
#from PropFair import *

'''
This file is edited from Ryu example which is located at  ryu/ryu/app/simple_switch.py.
According to its licecse(please don't trust my reading and read it), we can modify and use it as long as we keep the old license and state we've change the code. --Joe
'''

FLOW_HARD_TIMEOUT = 30
FLOW_IDLE_TIMEOUT = 10
CONTROLLER_IP = "10.10.4.1"
CONTROLLER_MAC = "00:00:00:00:00:01"

class SimpleSwitch(app_manager.RyuApp):
    global printComments

    OFP_VERSIONS = [ofproto_v1_0.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.ip_mac_port = {}  #collection of all interfaces connected to OVS
	printComments = 1
	self.configuration = {}  #table for the configuration: DEFAULT/IP_addess;{PARAM*WEIGHT};TIMEOUT
	self.number_of_servers = {}
	self.servers = {}
        self.serverLoad = [0]
	#read the configuration file
	__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
	with open(os.path.join(__location__, 'config')) as f:
           for line in f:
             if line.startswith( '#' ):
	       continue
	     split_line = re.split(';',line[:-1])
             server = split_line[0]
             parameters_weights = re.split(',',split_line[1])
             parameter_list = []
             for pw in parameters_weights:
               parameter = re.split('\*',pw)
               if len(parameter)==1:
                  #in case if we don't specify the weight
                  parameter.append(100/len(parameters_weights))
               parameter_list.append(tuple(parameter))
	     timeout = split_line[2]
	     self.configuration.setdefault(server, {})
             self.configuration[server] = (parameter_list, timeout)
	print self.configuration

        if "DEFAULT" not in self.configuration:
           print "WRONG CONFIGURATION"
	   exit()

        print "Configuration validated"

    def add_flow(self, datapath, match, act, priority=0, idle_timeout=0, flags=0, cookie=0):
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser
        mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, actions=act, flags=flags, idle_timeout=idle_timeout, cookie=cookie)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        print "Packet in"
        #parse the message from the event
        msg = ev.msg
	in_port = msg.in_port
        #switch id
        datapath = msg.datapath
        dpid = datapath.id
        ofproto = datapath.ofproto
        #set default key in a dictionary
        self.ip_mac_port.setdefault(dpid, {})

        #read the packet
        pkt = packet.Packet(msg.data)
	eth = pkt.get_protocol(ethernet.ethernet)
        dst = eth.dst
        src = eth.src

        arp_pkt = pkt.get_protocol(arp.arp)
        if arp_pkt:
          # self.logger.info("arp %s", arp_pkt)
           src_ip = arp_pkt.src_ip
           dst_ip = arp_pkt.dst_ip

           #save ip to mac and port correspondance
           self.ip_mac_port[dpid][src_ip] = (src, in_port)
           print self.ip_mac_port[dpid][src_ip]

           self.send_arp_reply(dpid, datapath, eth, arp_pkt, src_ip, dst_ip, in_port, msg)

        ipv4_pkt = pkt.get_protocol(ipv4.ipv4)
        if ipv4_pkt:
           self.logger.info("ipv4_packet %s", ipv4_pkt)
           self.ip_packet_handler(dpid, datapath, eth, msg, pkt, in_port, ipv4_pkt)

    def ip_packet_handler (self, dpid, datapath, eth, msg, pkt, in_port, ipv4_pkt):
	print "Got IP packet"
        udp_segment = pkt.get_protocol(udp.udp)
        tcp_segment = pkt.get_protocol(tcp.tcp)
        parser = datapath.ofproto_parser
	ofproto = datapath.ofproto
        if udp_segment and ipv4_pkt.dst == CONTROLLER_IP:
           #This packet is for controller
           self.server_controller_communication(dpid, datapath, eth, msg, in_port, ipv4_pkt, udp_segment)
        elif tcp_segment:
	   dl_type_ipv4 = 0x0800
           timeout = 60
           #start calling SCHEDULER
           serverID = 1
           #end calling SCHEDULER
           serverIP = self.servers[dpid][serverID][0]
           serverIP_int = ipv4_to_int(serverIP)
           serverPORT = self.ip_mac_port[dpid][serverIP][1]
           serverMAC = self.ip_mac_port[dpid][serverIP][0]
           self.serverLoad[serverID] += 1

           clientIP_int = ipv4_to_int(ipv4_pkt.src)
           clientPORT = in_port

	   print "Adding FORWARD Flow"
           match = parser.OFPMatch (dl_type = dl_type_ipv4, nw_src=clientIP_int,
                                                            tp_src=tcp_segment.src_port, nw_proto = 6) #TCP proto 6
           actions = [parser.OFPActionSetNwDst(serverIP_int),
                    parser.OFPActionSetDlDst(haddr_to_bin(serverMAC)), parser.OFPActionOutput(serverPORT)]
           self.add_flow(datapath=datapath, match=match, act=actions, priority=1, idle_timeout=timeout,
                        flags=ofproto.OFPFF_SEND_FLOW_REM, cookie=serverID)

	   print "Adding REVERSE Flow"
	   match = parser.OFPMatch (dl_type = dl_type_ipv4, nw_src=serverIP_int,
                                    nw_dst=clientIP_int, tp_dst=tcp_segment.src_port)
           actions = [ parser.OFPActionSetNwSrc (ipv4_to_int(ipv4_pkt.dst)), #REWRITE IP HEADER FOR TCP_HANDSHAKE
                    parser.OFPActionOutput(clientPORT)]
           self.add_flow(datapath=datapath, match=match, act=actions, priority=1, idle_timeout=timeout)

           #sending this packet
           print "Sending the packet out"
           actions = []
	   actions.append( createOFAction(datapath, ofproto.OFPAT_OUTPUT, serverPORT) )
           sendPacketOut(msg=msg, actions=actions, buffer_id=msg.buffer_id)

    def server_controller_communication(self, dpid, datapath, eth, msg, in_port, ipv4_pkt, udp_segment):
	self.logger.info("udp %s", udp_segment)
        print "Got UDP packet for controller"
        #parsing udp segment
        udp_pointer = len(msg.data) - udp_segment.total_length + 8
        message = msg.data[udp_pointer:]
        if udp_segment.dst_port == 7777 and message == "Hello":
          print "Server initialization"
          #send UDP reply with the list of parameters and timeout
          self.ip_mac_port[dpid][ipv4_pkt.src] = (eth.src, in_port)
	  if ipv4_pkt.src in self.configuration:
            conf_src = ipv4_pkt.src
          else:
            conf_src = "DEFAULT"

          serverID = findIPInServerList(self.servers[dpid], ipv4_pkt.src)
          if serverID==-1:
            print "Add new server"
            serverID = self.number_of_servers[dpid] = self.number_of_servers[dpid] + 1
            self.servers[dpid][serverID] = [ipv4_pkt.src,[]]
            self.servers[dpid][serverID][1]=[0.0]*len(self.configuration[conf_src][0])
            self.serverLoad.append(0)

          message = ','.join(p[0] for p in self.configuration[conf_src][0]) + ";" + self.configuration[conf_src][1] + ";" + str(serverID)
	  self.send_udp_reply(dpid, datapath, eth, ipv4_pkt, udp_segment, in_port, message)

          print self.servers

        elif udp_segment.dst_port == 7778:
          print message
          recieved_data = re.split(';', message)
	  serverID = int(recieved_data[1])
          if serverID not in self.servers[dpid]:
            print "Server not registered"
          else:
	    values = re.split(',', recieved_data[0])
            i = 0
            for value in values:
              self.servers[dpid][serverID][1][i] = float(value)
              i += 1
            print self.servers

        print self.ip_mac_port

    def send_udp_reply(self, dpid, datapath, eth, ipv4_pkt, udp_segment, out_port, message):
	ofproto = datapath.ofproto
        e = ethernet.ethernet(dst=eth.src, src=CONTROLLER_MAC, ethertype=ether.ETH_TYPE_IP)
        i = ipv4.ipv4(dst=ipv4_pkt.src, src=ipv4_pkt.dst, proto=17, total_length=0)
        u = udp.udp(src_port=udp_segment.dst_port, dst_port=udp_segment.src_port, total_length=0, csum=0)
        udp_h = u.serialize(message, i)
        ip_h = i.serialize(udp_h + message, e)
        eth_h = e.serialize(ip_h + udp_h + message, None)

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port)]
        data = eth_h + ip_h + udp_h + message

	out = datapath.ofproto_parser.OFPPacketOut(datapath=datapath, buffer_id=0xffffffff,
            in_port=datapath.ofproto.OFPP_CONTROLLER,
            actions=actions, data=data)
	datapath.send_msg(out)

    def send_arp_reply(self, dpid, datapath, eth, arp_pkt, src_ip, dst_ip, in_port, msg):
	ofproto = datapath.ofproto
        out_port = None

	if dst_ip == CONTROLLER_IP:
         #  print "Controller ARP request"
           e = ethernet.ethernet(dst=eth.src, src=CONTROLLER_MAC, ethertype=ether.ETH_TYPE_ARP)
           a = arp.arp(hwtype=1, proto=0x0800, hlen=6, plen=4, opcode=2, src_mac=CONTROLLER_MAC,
               src_ip=CONTROLLER_IP, dst_mac=eth.src, dst_ip=src_ip)

        elif dst_ip in self.ip_mac_port[dpid]:
           out_port = self.ip_mac_port[dpid][dst_ip][1]
        else:
           out_port = ofproto.OFPP_FLOOD

        if (out_port != None):
           actions = [datapath.ofproto_parser.OFPActionOutput(out_port)]
           data = msg.data

	else:
	   p = packet.Packet()
           p.add_protocol(e)
           p.add_protocol(a)
           p.serialize()
           actions = [datapath.ofproto_parser.OFPActionOutput(in_port)]
           data = p.data

	out = datapath.ofproto_parser.OFPPacketOut(datapath=datapath, buffer_id=0xffffffff,
            in_port=datapath.ofproto.OFPP_CONTROLLER,
            actions=actions, data=data)
	datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPFlowRemoved, MAIN_DISPATCHER)
    def flow_removal_handler(self, ev):
        msg = ev.msg

    def remove_table_flows(self, datapath, table_id, match):
        """Create OFP flow mod message to remove flows from table."""
        ofproto = datapath.ofproto
        flow_mod = datapath.ofproto_parser.OFPFlowMod(datapath=datapath, match=match, command=ofproto.OFPFC_DELETE,
                             cookie=0, idle_timeout=0,out_port=65535, buffer_id=4294967295, flags=0, hard_timeout=0,priority=0, actions=[])
        return flow_mod

    @set_ev_cls(dpset.EventDP, dpset.DPSET_EV_DISPATCHER)
    def _event_switch_enter_handler(self, ev):
       #definitions
       actions = []
       dl_type_ipv4 = 0x0800
       dl_type_arp = 0x0806
       dp = ev.dp
       dpid = dp.id

       ofproto = dp.ofproto
       parser = dp.ofproto_parser
       empty_match = parser.OFPMatch()
       instructions = []

       self.number_of_servers.setdefault(dpid, {})
       self.servers.setdefault(dpid, {})
       self.number_of_servers[dpid]=0
       self.servers[dpid][0]=('0')

       #empty the flow table
       flow_mod = self.remove_table_flows(dp, 0,empty_match)
       dp.send_msg(flow_mod)

       #process arp packets normally
       actions = [parser.OFPActionOutput(ofproto.OFPP_NORMAL)]
       match = parser.OFPMatch(dl_type = dl_type_arp)
#       self.add_flow(dp, match, actions, priority=100)

       #add miss flow
       match = parser.OFPMatch ()
       actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER)]
       self.add_flow(dp, match, actions, priority=0)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
from urllib.parse import urljoin
import os
import logging
from bs4 import BeautifulSoup
from utils import utils, inspector

# https://www.oig.dot.gov/
# oldest year: 1996

#
# options:
#   standard since/year options for a year range to fetch from.
#
#   topics - limit reports fetched to one or more topics, comma-separated.
#            e.g. "A,IN". Defaults to all topics.
#
#          A    - Audits
#          IN   - Investigations
#          T    - Testimony
#          C    - Correspondence
#          R    - Recovery Act
#          SA   - Semiannual Reports

# Notes to IG's web team:
#   - All report types are the same (some report links just link to other agency's homepages)
#   - No easy way to distinguish work product from preliminary and intermediary announcements.
#   - Add report ids to all pages in a consistent way
#   - From a report's landing page, no way to see what oversight area or sub-agency a report belongs to.
#   - Read https://github.com/unitedstates/inspectors-general/pull/50 for more details.

TOPIC_TO_URL = {
  "A": "https://www.oig.dot.gov/audits",
  "IN": "https://www.oig.dot.gov/investigations",
  "T": "https://www.oig.dot.gov/testimony",
  "C": "https://www.oig.dot.gov/correspondence",
  "R": "https://www.oig.dot.gov/recovery",
  "SA": "https://www.oig.dot.gov/semiannual-reports",
}

TYPES_WITHOUT_REPORTS = [
  'New Audit Announcements',
  'Testimony'
]

UNRELEASED_REPORT_TYPES = [
  'Investigations',
  'Audit Reports',
]

UNRELEASED_REPORT_IDS = [
  "CC-2000-290",
  "AV-2009-066"
]

LANDING_URLS_TO_REPORT_LINKS = {
  "https://www.oig.dot.gov/library-item/5235": "https://www.osc.gov/FY2010/Scanned/10-07%20DI-08-2225/DI-08-2225%20Agency%20Report%20Part%201%20(Revised).PDF",
}

BASE_REPORT_PAGE_URL = BASE_REPORT_URL = "https://www.oig.dot.gov/"

def run(options):
  year_range = inspector.year_range(options)

  topics = options.get('topics')
  if topics:
    topics = topics.split(",")
  else:
    topics = TOPIC_TO_URL.keys()

  for topic in topics:
    year_urls = urls_for(year_range, topic)
    for year_url in year_urls:
      body = utils.download(year_url)

      doc = BeautifulSoup(body)
      results = doc.select(".view-business-areas .views-row")

      for result in results:
        report = report_from(result, year_range, topic)
        if report:
          inspector.save_report(report)

def urls_for(year_range, topic):
  topic_url = TOPIC_TO_URL[topic]
  return ["%s?date_filter[value][year]=%s" % (topic_url, year) for year in year_range]

def report_from(result, year_range, topic):
  published_date_text = result.select('.date-display-single')[0].text
  published_on = datetime.datetime.strptime(published_date_text, "%m.%d.%Y")

  landing_url_link = result.select('a')[0]
  landing_url_relative = landing_url_link['href']
  title = landing_url_link.text
  landing_url = urljoin(BASE_REPORT_PAGE_URL, landing_url_relative)

  if published_on.year not in year_range:
    # Some reports come in from the 'Most Popular' section of the page and that
    # section does not obey the year filters
    logging.debug("[%s] Skipping, not in requested range." % landing_url)
    return

  logging.debug("### Processing report %s" % landing_url)

  report_page_body = utils.download(landing_url)
  report_page = BeautifulSoup(report_page_body)

  # take an expansive view of the 'summary' -
  #   landing page title, and any paragraphs with summary text
  summary = report_page.select(".node-library-item")[0].text.strip()

  # the text from the last link in the breadcrumb
  crumbs = report_page.select("nav.breadcrumb ol li")
  if len(crumbs) == 1:
    report_type = "Press Release"
  else:
    report_type = report_page.select("nav.breadcrumb ol li")[1].select("a")[0].text

  # should always work?
  report_id = landing_url.split("/")[-1]

  unreleased = False
  try:
    report_url = urljoin(BASE_REPORT_URL, report_page.select(".download-pdf a")[0]['href'])
  except IndexError as exc:
    if report_type in TYPES_WITHOUT_REPORTS:
      # Some types just don't have reports(Announcements), ignore them
      return
    elif (
        'For Official Use Only' in summary
        or report_type in UNRELEASED_REPORT_TYPES
        or report_id in UNRELEASED_REPORT_IDS
      ):
      unreleased = True
      report_url = None
    elif landing_url in LANDING_URLS_TO_REPORT_LINKS:
      report_url = LANDING_URLS_TO_REPORT_LINKS[landing_url]
    else:
      raise exc

  # Some investigations provide a report link, but they are just links to other
  # sites. Mark these as unreleased.
  # Ex. https://www.oig.dot.gov/library-item/5085
  if report_url:
    report_extension = os.path.splitext(os.path.basename(report_url))[1]
    if report_type == 'Investigations' and not report_extension:
      unreleased = True
      report_url = None

  # The DOT has lots of links to reports from other agencies. Ignore them.
  if report_url and BASE_REPORT_URL not in report_url:
    logging.debug("### Skipping report %s (not DOT)" % landing_url)
    return

  # These are just announcements, not real reports
  if "Audit Initiated" in title:
    return

  # Check for related reports. This will help us group related reports later.
  related_reports = [
    urljoin(BASE_REPORT_URL, link['href'])
    for link
    in report_page.select(".view-related-library-items li a")
  ]

  report = {
    'inspector': 'dot',
    'inspector_url': 'https://www.oig.dot.gov',
    'agency': 'dot',
    'agency_name': 'Department of Transportation',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
    'landing_url': landing_url,
    'summary': summary,
    'topic': topic,
    'type': report_type,
  }
  if unreleased:
    report['unreleased'] = unreleased
  if related_reports:
    report['related_reports'] = related_reports
  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[dot] add option to limit to individual report ID<commit_after>#!/usr/bin/env python

import datetime
from urllib.parse import urljoin
import os
import logging
from bs4 import BeautifulSoup
from utils import utils, inspector

# https://www.oig.dot.gov/
# oldest year: 1996

#
# options:
#   standard since/year options for a year range to fetch from.
#
#   topics - limit reports fetched to one or more topics, comma-separated.
#            e.g. "A,IN". Defaults to all topics.
#
#          A    - Audits
#          IN   - Investigations
#          T    - Testimony
#          C    - Correspondence
#          R    - Recovery Act
#          SA   - Semiannual Reports

# Notes to IG's web team:
#   - All report types are the same (some report links just link to other agency's homepages)
#   - No easy way to distinguish work product from preliminary and intermediary announcements.
#   - Add report ids to all pages in a consistent way
#   - From a report's landing page, no way to see what oversight area or sub-agency a report belongs to.
#   - Read https://github.com/unitedstates/inspectors-general/pull/50 for more details.

TOPIC_TO_URL = {
  "A": "https://www.oig.dot.gov/audits",
  "IN": "https://www.oig.dot.gov/investigations",
  "T": "https://www.oig.dot.gov/testimony",
  "C": "https://www.oig.dot.gov/correspondence",
  "R": "https://www.oig.dot.gov/recovery",
  "SA": "https://www.oig.dot.gov/semiannual-reports",
}

TYPES_WITHOUT_REPORTS = [
  'New Audit Announcements',
  'Testimony'
]

UNRELEASED_REPORT_TYPES = [
  'Investigations',
  'Audit Reports',
]

UNRELEASED_REPORT_IDS = [
  "CC-2000-290",
  "AV-2009-066"
]

LANDING_URLS_TO_REPORT_LINKS = {
  "https://www.oig.dot.gov/library-item/5235": "https://www.osc.gov/FY2010/Scanned/10-07%20DI-08-2225/DI-08-2225%20Agency%20Report%20Part%201%20(Revised).PDF",
}

BASE_REPORT_PAGE_URL = BASE_REPORT_URL = "https://www.oig.dot.gov/"

def run(options):
  year_range = inspector.year_range(options)

  topics = options.get('topics')
  if topics:
    topics = topics.split(",")
  else:
    topics = TOPIC_TO_URL.keys()

  for topic in topics:
    year_urls = urls_for(year_range, topic)
    for year_url in year_urls:
      body = utils.download(year_url)

      doc = BeautifulSoup(body)
      results = doc.select(".view-business-areas .views-row")

      for result in results:
        report = report_from(result, year_range, topic, options)
        if report:
          inspector.save_report(report)

def urls_for(year_range, topic):
  topic_url = TOPIC_TO_URL[topic]
  return ["%s?date_filter[value][year]=%s" % (topic_url, year) for year in year_range]

def report_from(result, year_range, topic, options):
  published_date_text = result.select('.date-display-single')[0].text
  published_on = datetime.datetime.strptime(published_date_text, "%m.%d.%Y")

  landing_url_link = result.select('a')[0]
  landing_url_relative = landing_url_link['href']

  # scan to see if we're limiting to a single report_id in dev
  initial_report_id = landing_url_relative.split("/")[-1]
  only_report_id = options.get('report_id')
  if only_report_id and (initial_report_id != only_report_id):
    logging.debug("[%s] Skipping, not the requested report" % initial_report_id)
    return

  title = landing_url_link.text
  landing_url = urljoin(BASE_REPORT_PAGE_URL, landing_url_relative)

  if published_on.year not in year_range:
    # Some reports come in from the 'Most Popular' section of the page and that
    # section does not obey the year filters
    logging.debug("[%s] Skipping, not in requested range." % landing_url)
    return

  logging.debug("### Processing report %s" % landing_url)

  report_page_body = utils.download(landing_url)
  report_page = BeautifulSoup(report_page_body)

  # take an expansive view of the 'summary' -
  #   landing page title, and any paragraphs with summary text
  summary = report_page.select(".node-library-item")[0].text.strip()

  # the text from the last link in the breadcrumb
  crumbs = report_page.select("nav.breadcrumb ol li")
  if len(crumbs) == 1:
    report_type = "Press Release"
  else:
    report_type = report_page.select("nav.breadcrumb ol li")[1].select("a")[0].text

  # should always work?
  report_id = landing_url.split("/")[-1]

  unreleased = False
  try:
    report_url = urljoin(BASE_REPORT_URL, report_page.select(".download-pdf a")[0]['href'])
  except IndexError as exc:
    if report_type in TYPES_WITHOUT_REPORTS:
      # Some types just don't have reports(Announcements), ignore them
      return
    elif (
        'For Official Use Only' in summary
        or report_type in UNRELEASED_REPORT_TYPES
        or report_id in UNRELEASED_REPORT_IDS
      ):
      unreleased = True
      report_url = None
    elif landing_url in LANDING_URLS_TO_REPORT_LINKS:
      report_url = LANDING_URLS_TO_REPORT_LINKS[landing_url]
    else:
      raise exc

  # Some investigations provide a report link, but they are just links to other
  # sites. Mark these as unreleased.
  # Ex. https://www.oig.dot.gov/library-item/5085
  if report_url:
    report_extension = os.path.splitext(os.path.basename(report_url))[1]
    if report_type == 'Investigations' and not report_extension:
      unreleased = True
      report_url = None

  # The DOT has lots of links to reports from other agencies. Ignore them.
  if report_url and BASE_REPORT_URL not in report_url:
    logging.debug("### Skipping report %s (not DOT)" % landing_url)
    return

  # These are just announcements, not real reports
  if "Audit Initiated" in title:
    return

  # Check for related reports. This will help us group related reports later.
  related_reports = [
    urljoin(BASE_REPORT_URL, link['href'])
    for link
    in report_page.select(".view-related-library-items li a")
  ]

  report = {
    'inspector': 'dot',
    'inspector_url': 'https://www.oig.dot.gov',
    'agency': 'dot',
    'agency_name': 'Department of Transportation',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
    'landing_url': landing_url,
    'summary': summary,
    'topic': topic,
    'type': report_type,
  }
  if unreleased:
    report['unreleased'] = unreleased
  if related_reports:
    report['related_reports'] = related_reports
  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>