<commit_before>import warnings
import pandas as pd
import time
import logging as lg

from urbanaccess.utils import log
from urbanaccess.gtfs.utils_validation import _check_time_range_format
from urbanaccess.gtfs.network import _time_selector

warnings.simplefilter(action="ignore", category=FutureWarning)

def _calc_headways_by_route_stop(df):
    """
    Calculate headways by route stop

    Parameters
    ----------
    df : pandas.DataFrame
        interpolated stop times DataFrame for stop times within the time
        range with appended trip and route information

    Returns
    -------
    DataFrame : pandas.DataFrame
        DataFrame of statistics of route stop headways in units of minutes
    """

    # TODO: Optimize for speed

    start_time = time.time()

    df['unique_stop_route'] = (
        df['unique_stop_id'].str.cat(
            df['unique_route_id'].astype('str'), sep=','))

    stop_route_groups = df.groupby('unique_stop_route')
    log('Starting route stop headway calculation for {:,} route '
        'stops...'.format(len(stop_route_groups)))

    results = {}

    # suppress RuntimeWarning: Mean of empty slice. for this code block
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category='RuntimeWarning')

        for unique_stop_route, stop_route_group in stop_route_groups:
            stop_route_group.sort_values(['departure_time_sec_interpolate'],
                                         ascending=True, inplace=True)
            next_bus_time = (stop_route_group['departure_time_sec_interpolate']
                             .iloc[1:].values)
            prev_bus_time = (stop_route_group['departure_time_sec_interpolate']
                             .iloc[:-1].values)
            stop_route_group_headways = (next_bus_time - prev_bus_time) / 60
            results[unique_stop_route] = (pd.Series(stop_route_group_headways)
                                          .describe())

    log('Route stop headway calculation complete. Took {:,.2f} seconds'.format(
        time.time() - start_time))

    return pd.DataFrame(results).T

def _headway_handler(interpolated_stop_times_df, trips_df,
                     routes_df, headway_timerange):
    """
    route stop headway calculator handler

    Parameters
    ----------
    interpolated_stop_times_df : pandas.DataFrame
        interpolated stop times DataFrame for stop times within the time range
    trips_df : pandas.DataFrame
        trips DataFrame
    routes_df : pandas.DataFrame
        routes DataFrame
    headway_timerange : list
        time range for which to calculate headways between in a list with time
        1 and time 2 as strings. Must follow format of a 24 hour clock for
        example: 08:00:00 or 17:00:00

    Returns
    -------
    headway_by_routestop_df : pandas.DataFrame
        DataFrame of statistics of route stop headways in units of minutes
        with relevant route and stop information
    """
    start_time = time.time()

    # add unique trip and route ID
    trips_df['unique_trip_id'] = (
        trips_df['trip_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))
    trips_df['unique_route_id'] = (
        trips_df['route_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id', 'service_id', 'unique_trip_id',
               'unique_agency_id']
    # if these optional cols exist then keep those that do
    optional_cols = ['direction_id', 'shape_id']
    for item in optional_cols:
        if item in trips_df.columns:
            columns.append(item)

    trips_df = trips_df[columns]

    # add unique route ID
    routes_df['unique_route_id'] = (
        routes_df['route_id'].str.cat(
            routes_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id', 'route_long_name', 'route_type',
               'unique_agency_id']
    routes_df = routes_df[columns]

    selected_interpolated_stop_times_df = _time_selector(
        df=interpolated_stop_times_df, starttime=headway_timerange[0],
        endtime=headway_timerange[1])

    tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id',
                    right_on='unique_route_id', sort=False)
    merge_df = pd.merge(selected_interpolated_stop_times_df, tmp1, how='left',
                        left_on='unique_trip_id', right_on='unique_trip_id',
                        sort=False)
    cols_to_drop = ['unique_agency_id_y', 'unique_agency_id_x']
    merge_df.drop(cols_to_drop, axis=1, inplace=True)

    headway_by_routestop_df = _calc_headways_by_route_stop(df=merge_df)

    # add unique route stop node_id
    headway_by_routestop_df = pd.merge(
        headway_by_routestop_df,
        merge_df[['unique_stop_route', 'unique_stop_id', 'unique_route_id']],
        how='left', left_index=True, right_on='unique_stop_route', sort=False)
    headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True)
    headway_by_routestop_df['node_id_route'] = (
        headway_by_routestop_df['unique_stop_id'].str.cat(
            headway_by_routestop_df['unique_route_id'].astype('str'), sep='_'))

    log('Headway calculation complete. Took {:,.2f} seconds.'.format(
        time.time() - start_time))

    return headway_by_routestop_df

def headways(gtfsfeeds_df, headway_timerange):
    """
    Calculate headways by route stop for a specific time range

    Parameters
    ----------
    gtfsfeeds_df : object
        gtfsfeeds_dfs object with all processed GTFS data tables
    headway_timerange : list
        time range for which to calculate headways between in a list with time
        1 and time 2 as strings. Must follow format of a 24 hour clock for
        example: 08:00:00 or 17:00:00

    Returns
    -------
    gtfsfeeds_dfs.headways : pandas.DataFrame
        gtfsfeeds_dfs object for the headways DataFrame with statistics of
        route stop headways in units of minutes with relevant route and stop
        information
    """
    _check_time_range_format(headway_timerange)

    if gtfsfeeds_df is None:
        raise ValueError('gtfsfeeds_df cannot be None.')
    if gtfsfeeds_df.stop_times_int.empty or gtfsfeeds_df.trips.empty or \
            gtfsfeeds_df.routes.empty:
        raise ValueError(
            'One of the following gtfsfeeds_dfs objects: stop_times_int, '
            'trips, or routes were found to be empty.')

    headways_df = _headway_handler(
        interpolated_stop_times_df=gtfsfeeds_df.stop_times_int,
        trips_df=gtfsfeeds_df.trips,
        routes_df=gtfsfeeds_df.routes,
        headway_timerange=headway_timerange)

    gtfsfeeds_df.headways = headways_df

    return gtfsfeeds_df
<commit_msg>remove unused import<commit_after>import warnings
import pandas as pd
import time

from urbanaccess.utils import log
from urbanaccess.gtfs.utils_validation import _check_time_range_format
from urbanaccess.gtfs.network import _time_selector

warnings.simplefilter(action="ignore", category=FutureWarning)

def _calc_headways_by_route_stop(df):
    """
    Calculate headways by route stop

    Parameters
    ----------
    df : pandas.DataFrame
        interpolated stop times DataFrame for stop times within the time
        range with appended trip and route information

    Returns
    -------
    DataFrame : pandas.DataFrame
        DataFrame of statistics of route stop headways in units of minutes
    """

    # TODO: Optimize for speed

    start_time = time.time()

    df['unique_stop_route'] = (
        df['unique_stop_id'].str.cat(
            df['unique_route_id'].astype('str'), sep=','))

    stop_route_groups = df.groupby('unique_stop_route')
    log('Starting route stop headway calculation for {:,} route '
        'stops...'.format(len(stop_route_groups)))

    results = {}

    # suppress RuntimeWarning: Mean of empty slice. for this code block
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category='RuntimeWarning')

        for unique_stop_route, stop_route_group in stop_route_groups:
            stop_route_group.sort_values(['departure_time_sec_interpolate'],
                                         ascending=True, inplace=True)
            next_bus_time = (stop_route_group['departure_time_sec_interpolate']
                             .iloc[1:].values)
            prev_bus_time = (stop_route_group['departure_time_sec_interpolate']
                             .iloc[:-1].values)
            stop_route_group_headways = (next_bus_time - prev_bus_time) / 60
            results[unique_stop_route] = (pd.Series(stop_route_group_headways)
                                          .describe())

    log('Route stop headway calculation complete. Took {:,.2f} seconds'.format(
        time.time() - start_time))

    return pd.DataFrame(results).T

def _headway_handler(interpolated_stop_times_df, trips_df,
                     routes_df, headway_timerange):
    """
    route stop headway calculator handler

    Parameters
    ----------
    interpolated_stop_times_df : pandas.DataFrame
        interpolated stop times DataFrame for stop times within the time range
    trips_df : pandas.DataFrame
        trips DataFrame
    routes_df : pandas.DataFrame
        routes DataFrame
    headway_timerange : list
        time range for which to calculate headways between in a list with time
        1 and time 2 as strings. Must follow format of a 24 hour clock for
        example: 08:00:00 or 17:00:00

    Returns
    -------
    headway_by_routestop_df : pandas.DataFrame
        DataFrame of statistics of route stop headways in units of minutes
        with relevant route and stop information
    """
    start_time = time.time()

    # add unique trip and route ID
    trips_df['unique_trip_id'] = (
        trips_df['trip_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))
    trips_df['unique_route_id'] = (
        trips_df['route_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id', 'service_id', 'unique_trip_id',
               'unique_agency_id']
    # if these optional cols exist then keep those that do
    optional_cols = ['direction_id', 'shape_id']
    for item in optional_cols:
        if item in trips_df.columns:
            columns.append(item)

    trips_df = trips_df[columns]

    # add unique route ID
    routes_df['unique_route_id'] = (
        routes_df['route_id'].str.cat(
            routes_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id', 'route_long_name', 'route_type',
               'unique_agency_id']
    routes_df = routes_df[columns]

    selected_interpolated_stop_times_df = _time_selector(
        df=interpolated_stop_times_df, starttime=headway_timerange[0],
        endtime=headway_timerange[1])

    tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id',
                    right_on='unique_route_id', sort=False)
    merge_df = pd.merge(selected_interpolated_stop_times_df, tmp1, how='left',
                        left_on='unique_trip_id', right_on='unique_trip_id',
                        sort=False)
    cols_to_drop = ['unique_agency_id_y', 'unique_agency_id_x']
    merge_df.drop(cols_to_drop, axis=1, inplace=True)

    headway_by_routestop_df = _calc_headways_by_route_stop(df=merge_df)

    # add unique route stop node_id
    headway_by_routestop_df = pd.merge(
        headway_by_routestop_df,
        merge_df[['unique_stop_route', 'unique_stop_id', 'unique_route_id']],
        how='left', left_index=True, right_on='unique_stop_route', sort=False)
    headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True)
    headway_by_routestop_df['node_id_route'] = (
        headway_by_routestop_df['unique_stop_id'].str.cat(
            headway_by_routestop_df['unique_route_id'].astype('str'), sep='_'))

    log('Headway calculation complete. Took {:,.2f} seconds.'.format(
        time.time() - start_time))

    return headway_by_routestop_df

def headways(gtfsfeeds_df, headway_timerange):
    """
    Calculate headways by route stop for a specific time range

    Parameters
    ----------
    gtfsfeeds_df : object
        gtfsfeeds_dfs object with all processed GTFS data tables
    headway_timerange : list
        time range for which to calculate headways between in a list with time
        1 and time 2 as strings. Must follow format of a 24 hour clock for
        example: 08:00:00 or 17:00:00

    Returns
    -------
    gtfsfeeds_dfs.headways : pandas.DataFrame
        gtfsfeeds_dfs object for the headways DataFrame with statistics of
        route stop headways in units of minutes with relevant route and stop
        information
    """
    _check_time_range_format(headway_timerange)

    if gtfsfeeds_df is None:
        raise ValueError('gtfsfeeds_df cannot be None.')
    if gtfsfeeds_df.stop_times_int.empty or gtfsfeeds_df.trips.empty or \
            gtfsfeeds_df.routes.empty:
        raise ValueError(
            'One of the following gtfsfeeds_dfs objects: stop_times_int, '
            'trips, or routes were found to be empty.')

    headways_df = _headway_handler(
        interpolated_stop_times_df=gtfsfeeds_df.stop_times_int,
        trips_df=gtfsfeeds_df.trips,
        routes_df=gtfsfeeds_df.routes,
        headway_timerange=headway_timerange)

    gtfsfeeds_df.headways = headways_df

    return gtfsfeeds_df
<|endoftext|>


<commit_before>import re, inspect
from django.core.exceptions import ViewDoesNotExist
from django.core.urlresolvers import RegexURLPattern, RegexURLResolver

def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None, ignored_modules=None):
    """
    Return a list of views from a list of urlpatterns.

    Each object in the returned list is a tuple: (view_func, regex, name)
    """
    ignored_modules = ignored_modules if ignored_modules else []
    views = []
    for p in urlpatterns:
        if isinstance(p, RegexURLPattern):
            # Handle correct single URL patterns
            try:
                if namespace:
                    name = '{0}:{1}'.format(namespace, p.name)
                else:
                    name = p.name
                if hasattr(p.callback, '__module__'):
                    if p.callback.__module__.split('.')[0] not in ignored_modules:
                        views.append((p.callback, base + p.regex.pattern, name))
                else:
                    views.append((p.callback, base + p.regex.pattern, name))
            except ViewDoesNotExist:
                continue

        elif isinstance(p, RegexURLResolver):
            # Handle include() definitions
            try:
                patterns = p.url_patterns
            except ImportError:
                continue
            views.extend(extract_views_from_urlpatterns(patterns, base + p.regex.pattern,
                namespace=(namespace or p.namespace), ignored_modules=ignored_modules))

        elif hasattr(p, '_get_callback'):
            # Handle string like 'foo.views.view_name' or just function view
            try:
                views.append((p._get_callback(), base + p.regex.pattern, p.name))
            except ViewDoesNotExist:
                continue

        elif hasattr(p, 'url_patterns') or hasattr(p, '_get_url_patterns'):
            # Handle url_patterns objects
            try:
                patterns = p.url_patterns
            except ImportError:
                continue
            views.extend(extract_views_from_urlpatterns(patterns, base + p.regex.pattern,
                namespace=namespace, ignored_modules=ignored_modules))
        else:
            raise TypeError("%s does not appear to be a urlpattern object" % p)
    return views

def display_as_table(views):
    """
        Get list of views from dumpurls security management command
        and returns them in the form of table to print in command line
    """
    views = [row.split('||', 3) for row in sorted(views)]
    widths = [len(max(columns, key=len)) for columns in zip(*views)]
    widths = [width  if width < 100 else 100 for width in widths]
    table_views = []

    header = ('URL', 'Handler path', 'Decorators & Mixins')
    table_views.append(
        ' | '.join('{0:<{1}}'.format(title, width) for width, title in zip(widths, header))
    )
    table_views.append('-+-'.join('-' * width for width in widths))

    for row in views:
        if len(row[2]) > 100:
            row[2] = row[2].split(',')
            row[2] = [",".join(row[2][i:i+2]) for i in range(0, len(row[2]), 2)]

        mixins = row[2]
        if type(mixins) == list:
            i = 0
            for line in mixins:
                row[2] = line.strip()
                if i > 0:
                    row[0] = ''
                    row[1] = ''
                table_views.append(
                    ' | '.join('{0:<{1}}'.format(cdata, width) for width, cdata in zip(widths, row))
                )
                i += 1
        else:
            table_views.append(
                ' | '.join('{0:<{1}}'.format(cdata, width) for width, cdata in zip(widths, row))
            )

    return "\n".join([v for v in table_views]) + "\n"

def get_func_name(func):
    if hasattr(func, 'func_name'):
        return func.func_name
    elif hasattr(func, '__name__'):
        return func.__name__
    elif hasattr(func, '__class__'):
        return '%s()' % func.__class__.__name__
    else:
        return re.sub(r' at 0x[0-9a-f]+', '', repr(func))

def get_decorators(func):
    """
        Get function or class and return names of applied decorators
    """
    decorators = []
    if hasattr(func, '__module__'):
        mod = inspect.getmodule(func)
        source_code = inspect.getsourcelines(mod)[0]
        func_name = get_func_name(func)
        i = 0

        for line in source_code:
            if line.startswith('def {}'.format(func_name)) or line.startswith('class {}'.format(func_name)):
                j = 1
                k = source_code[i-j]
                while k.startswith('@'):
                    decorators.append(k.strip().split('(')[0])
                    j += 1
                    k = source_code[i-j]
            i += 1

    return decorators

def get_mixins(func, ignored_modules=None):
    """
        Get class and return names and paths to applied mixins
        Has an optional argument for names of modules that should be ignored
    """
    ignored_modules = ignored_modules if ignored_modules else []
    mixins = []
    if hasattr(func, 'cls'):
        for klass in func.cls.mro():
            if klass != func.cls and klass.__module__.split('.')[0] not in ignored_modules:
                mixins.append("{}.{}".format(klass.__module__, get_func_name(klass)))

    return mixins
<commit_msg>dumpurls: Avoiding re-defining the same strings on every iteration of a for loop.<commit_after>import re, inspect
from django.core.exceptions import ViewDoesNotExist
from django.core.urlresolvers import RegexURLPattern, RegexURLResolver

def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None, ignored_modules=None):
    """
    Return a list of views from a list of urlpatterns.

    Each object in the returned list is a tuple: (view_func, regex, name)
    """
    ignored_modules = ignored_modules if ignored_modules else []
    views = []
    for p in urlpatterns:
        if isinstance(p, RegexURLPattern):
            # Handle correct single URL patterns
            try:
                if namespace:
                    name = '{0}:{1}'.format(namespace, p.name)
                else:
                    name = p.name
                if hasattr(p.callback, '__module__'):
                    if p.callback.__module__.split('.')[0] not in ignored_modules:
                        views.append((p.callback, base + p.regex.pattern, name))
                else:
                    views.append((p.callback, base + p.regex.pattern, name))
            except ViewDoesNotExist:
                continue

        elif isinstance(p, RegexURLResolver):
            # Handle include() definitions
            try:
                patterns = p.url_patterns
            except ImportError:
                continue
            views.extend(extract_views_from_urlpatterns(patterns, base + p.regex.pattern,
                namespace=(namespace or p.namespace), ignored_modules=ignored_modules))

        elif hasattr(p, '_get_callback'):
            # Handle string like 'foo.views.view_name' or just function view
            try:
                views.append((p._get_callback(), base + p.regex.pattern, p.name))
            except ViewDoesNotExist:
                continue

        elif hasattr(p, 'url_patterns') or hasattr(p, '_get_url_patterns'):
            # Handle url_patterns objects
            try:
                patterns = p.url_patterns
            except ImportError:
                continue
            views.extend(extract_views_from_urlpatterns(patterns, base + p.regex.pattern,
                namespace=namespace, ignored_modules=ignored_modules))
        else:
            raise TypeError("%s does not appear to be a urlpattern object" % p)
    return views

def display_as_table(views):
    """
        Get list of views from dumpurls security management command
        and returns them in the form of table to print in command line
    """
    views = [row.split('||', 3) for row in sorted(views)]
    widths = [len(max(columns, key=len)) for columns in zip(*views)]
    widths = [width  if width < 100 else 100 for width in widths]
    table_views = []

    header = ('URL', 'Handler path', 'Decorators & Mixins')
    table_views.append(
        ' | '.join('{0:<{1}}'.format(title, width) for width, title in zip(widths, header))
    )
    table_views.append('-+-'.join('-' * width for width in widths))

    for row in views:
        if len(row[2]) > 100:
            row[2] = row[2].split(',')
            row[2] = [",".join(row[2][i:i+2]) for i in range(0, len(row[2]), 2)]

        mixins = row[2]
        if type(mixins) == list:
            i = 0
            for line in mixins:
                row[2] = line.strip()
                if i > 0:
                    row[0] = ''
                    row[1] = ''
                table_views.append(
                    ' | '.join('{0:<{1}}'.format(cdata, width) for width, cdata in zip(widths, row))
                )
                i += 1
        else:
            table_views.append(
                ' | '.join('{0:<{1}}'.format(cdata, width) for width, cdata in zip(widths, row))
            )

    return "\n".join([v for v in table_views]) + "\n"

def get_func_name(func):
    if hasattr(func, 'func_name'):
        return func.func_name
    elif hasattr(func, '__name__'):
        return func.__name__
    elif hasattr(func, '__class__'):
        return '%s()' % func.__class__.__name__
    else:
        return re.sub(r' at 0x[0-9a-f]+', '', repr(func))

def get_decorators(func):
    """
        Get function or class and return names of applied decorators
    """
    decorators = []
    if hasattr(func, '__module__'):
        mod = inspect.getmodule(func)
        source_code = inspect.getsourcelines(mod)[0]
        func_name = get_func_name(func)
        i = 0

        func_def = 'def {}'.format(func_name)
        class_def = 'class {}'.format(func_name)

        for line in source_code:
            if line.startswith(func_def) or line.startswith(class_def):
                j = 1
                k = source_code[i-j]
                while k.startswith('@'):
                    decorators.append(k.strip().split('(')[0])
                    j += 1
                    k = source_code[i-j]
            i += 1

    return decorators

def get_mixins(func, ignored_modules=None):
    """
        Get class and return names and paths to applied mixins
        Has an optional argument for names of modules that should be ignored
    """
    ignored_modules = ignored_modules if ignored_modules else []
    mixins = []
    if hasattr(func, 'cls'):
        for klass in func.cls.mro():
            if klass != func.cls and klass.__module__.split('.')[0] not in ignored_modules:
                mixins.append("{}.{}".format(klass.__module__, get_func_name(klass)))

    return mixins
<|endoftext|>


<commit_before>import argparse
import os
import subprocess

import pandas as pd

import mummer_two_bins

def get_all_bin_paths():
    bin_info = pd.read_csv('./support_files/available_bins.csv')
    return bin_info['bin path']

def query_bin_file_to_list(filepath):
    print('load paths at {}'.format(filepath))
    with open(filepath) as f:
        lines = f.read().splitlines()
    return lines

def analyze_bin_pairs(partial_bins_list, results_dir,
                          preserve_existing=True):
    """
    loop over a list of some bin paths, and run MUMMER with every other bin
     as the reference.  Also parses the .coords file.

    :return: nothing to terminal/python but saves files
    """
    reference_bin_paths = get_all_bin_paths()

    for query_bin_path in partial_bins_list:

        # There will be a folder for each bin, and the MUMmer results for all
        # other bins as reference within
        result_sub_dir = mummer_two_bins.strip_off_fasta_suffix(
            os.path.basename(query_bin_path))
        results_dir = results_dir + "/" + result_sub_dir

        print("save result in {}".format(results_dir))

        # loop over all the bins, including itself.
        for ref_bin_path in reference_bin_paths:
            # first check whether the file exists:

            expected_coords_path = \
                mummer_two_bins.file_prefix_from_fata_paths(query_bin_path,
                                                            ref_bin_path,
                                                            results_dir) + \
                '.coords'
            coords_exists = os.path.exists(expected_coords_path)

            # Command to send to shell:
            command = ['python', "./support_files/mummer_two_bins.py",
                      query_bin_path, ref_bin_path, results_dir]

            if preserve_existing:

                if coords_exists:
                    print("not writing over existing mummer results: "
                    "{}".format(expected_coords_path))
                    # don't remake the file if it already exists.
                    continue
                else:
                    # Run mummer_two_bins on the pair of fastas.
                    print('mummer comamnd to run: \n`{}`'.format(
                        ' '.join(command)))
                    subprocess.check_call(command)

            # If we don't care whether the file exists or not, make it
            # fresh every time.
            else:
                subprocess.check_call(command)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description='Run MUMmer using a subset of query bins, and all of the'
                    'reference bins.')

    parser.add_argument("query_bin_path_list", type=str,
                        help='path to list of paths for query bins')
    parser.add_argument("results_dir", type=str,
                        help='path to save results to')
    parser.add_argument("preserve_existing", type=bool, default=True,
                        help='re-run Mummer even if file exists?')

    args = parser.parse_args()

    print(args)

    query_bins_as_list = query_bin_file_to_list(args.query_bin_path_list)

    analyze_bin_pairs(
        partial_bins_list=query_bins_as_list,
        results_dir=args.results_dir,
        preserve_existing=True)
<commit_msg>fixed accidental nesting of mummer results<commit_after>import argparse
import os
import subprocess

import pandas as pd

import mummer_two_bins

def get_all_bin_paths():
    bin_info = pd.read_csv('./support_files/available_bins.csv')
    return bin_info['bin path']

def query_bin_file_to_list(filepath):
    print('load paths at {}'.format(filepath))
    with open(filepath) as f:
        lines = f.read().splitlines()
    return lines

def analyze_bin_pairs(partial_bins_list, results_dir,
                          preserve_existing=True):
    """
    loop over a list of some bin paths, and run MUMMER with every other bin
     as the reference.  Also parses the .coords file.

    :return: nothing to terminal/python but saves files
    """
    reference_bin_paths = get_all_bin_paths()

    for query_bin_path in partial_bins_list:

        # There will be a folder for each bin, and the MUMmer results for all
        # other bins as reference within
        result_sub_dir = mummer_two_bins.strip_off_fasta_suffix(
            os.path.basename(query_bin_path))
        particular_results_dir = results_dir + "/" + result_sub_dir

        print("save result in {}".format(results_dir))

        # loop over all the bins, including itself.
        for ref_bin_path in reference_bin_paths:
            # first check whether the file exists:

            expected_coords_path = \
                mummer_two_bins.file_prefix_from_fata_paths(
                    query_bin_path, ref_bin_path, particular_results_dir) + \
                '.coords'
            coords_exists = os.path.exists(expected_coords_path)

            # Command to send to shell:
            command = ['python', "./support_files/mummer_two_bins.py",
                      query_bin_path, ref_bin_path, particular_results_dir]

            if preserve_existing:

                if coords_exists:
                    print("not writing over existing mummer results: "
                    "{}".format(expected_coords_path))
                    # don't remake the file if it already exists.
                    continue
                else:
                    # Run mummer_two_bins on the pair of fastas.
                    print('mummer comamnd to run: \n`{}`'.format(
                        ' '.join(command)))
                    subprocess.check_call(command)

            # If we don't care whether the file exists or not, make it
            # fresh every time.
            else:
                subprocess.check_call(command)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description='Run MUMmer using a subset of query bins, and all of the'
                    'reference bins.')

    parser.add_argument("query_bin_path_list", type=str,
                        help='path to list of paths for query bins')
    parser.add_argument("results_dir", type=str,
                        help='path to save results to')
    parser.add_argument("preserve_existing", type=bool, default=True,
                        help='re-run Mummer even if file exists?')

    args = parser.parse_args()

    print(args)

    query_bins_as_list = query_bin_file_to_list(args.query_bin_path_list)

    analyze_bin_pairs(
        partial_bins_list=query_bins_as_list,
        results_dir=args.results_dir,
        preserve_existing=True)
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright © 2015 The SublimeLinter Community
#
# License: MIT
#

"""This module exports the HtmlTidy plugin class."""

from SublimeLinter.lint import Linter, util

class HtmlTidy(Linter):

    """Provides an interface to tidy."""

    syntax = 'html'
    cmd = 'tidy -errors -quiet -utf8'
    regex = r'^line (?P<line>\d+) column (?P<col>\d+) - (?:(?P<error>Error)|(?P<warning>Warning)): (?P<message>.+)'
    error_stream = util.STREAM_STDERR
<commit_msg>checks if tidy5 (tidy-html5) is installed with different executable<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright © 2015 The SublimeLinter Community
#
# License: MIT
#

"""This module exports the HtmlTidy plugin class."""

from SublimeLinter.lint import Linter, util
import shutil

class HtmlTidy(Linter):

    """Provides an interface to tidy."""

    syntax = 'html'
    if shutils.which('tidy5'):
        cmd = 'tidy5 -errors -quiet -utf8'
    else:
        cmd = 'tidy -errors -quiet -utf8'
    regex = r'^line (?P<line>\d+) column (?P<col>\d+) - (?:(?P<error>Error)|(?P<warning>Warning)): (?P<message>.+)'
    error_stream = util.STREAM_STDERR
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# pysal documentation build configuration file, created by
# sphinx-quickstart on Wed Aug 26 19:58:20 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('sphinxext'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest','sphinx.ext.graphviz',
              'sphinx.ext.intersphinx', 'sphinx.ext.pngmath',
              'sphinx.ext.autosummary','numpydoc']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
#source_suffix = '.rst'
source_suffix = '.txt'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pysal'
copyright = u'2009-10, Sergio Rey'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.9.0'
# The full version, including alpha/beta/rc tags.
release = '0.9.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
#html_theme = 'default'
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = "%s v%s Reference Guide (DRAFT)" % (project, version)

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'pysaldoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pysal.tex', u'pysal Documentation',
   u'Serge Rey', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

#numpydoc option
numpydoc_show_class_members = True
<commit_msg>- version bump for docs<commit_after># -*- coding: utf-8 -*-
#
# pysal documentation build configuration file, created by
# sphinx-quickstart on Wed Aug 26 19:58:20 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('sphinxext'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest','sphinx.ext.graphviz',
              'sphinx.ext.intersphinx', 'sphinx.ext.pngmath',
              'sphinx.ext.autosummary','numpydoc']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
#source_suffix = '.rst'
source_suffix = '.txt'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pysal'
copyright = u'2009-10, Sergio Rey'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0.0'
# The full version, including alpha/beta/rc tags.
release = '1.0.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
#html_theme = 'default'
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = "%s v%s Reference Guide (DRAFT)" % (project, version)

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'pysaldoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pysal.tex', u'pysal Documentation',
   u'Serge Rey', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

#numpydoc option
numpydoc_show_class_members = True
<|endoftext|>


<commit_before># Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.

# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

# snippet-comment:[These are tags for the AWS doc team's sample catalog. Do not remove.]
# snippet-sourcedescription:[thing_performance.py demonstrates how to push CPU and memory usage data to a thing's device shadow in AWS IoT.]
# snippet-service:[iot]
# snippet-sourcesyntax:[python]
# snippet-keyword:[Python]
# snippet-keyword:[AWS IoT]
# snippet-keyword:[Code Sample]
# snippet-keyword:[AWSIoTSDK]
# snippet-keyword:[AWSIoTMQTTShadowClient]
# snippet-sourcetype:[full-example]
# snippet-sourcedate:[2020-01-23]
# snippet-sourceauthor:[FThompsonAWS]
# snippet-start:[iot.python.thing_performance.complete]

from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTShadowClient

import json
import psutil
import argparse
import logging
import time

def configureParser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-e", "--endpoint", action="store", required=True, dest="host",
            help="Your AWS IoT custom endpoint")
    parser.add_argument("-r", "--rootCA", action="store", required=True, dest="rootCAPath", help="Root CA file path")
    parser.add_argument("-c", "--cert", action="store", required=True, dest="certificatePath",
            help="Certificate file path")
    parser.add_argument("-k", "--key", action="store", required=True, dest="privateKeyPath",
            help="Private key file path")
    parser.add_argument("-p", "--port", action="store", dest="port", type=int, default=8883,
            help="Port number override")
    parser.add_argument("-n", "--thingName", action="store", required=True, dest="thingName",
            help="Targeted thing name")
    parser.add_argument("-d", "--requestDelay", action="store", dest="requestDelay", type=float, default=1,
            help="Time between requests (in seconds)")
    parser.add_argument("-v", "--enableLogging", action="store_true", dest="enableLogging",
            help="Enable AWS IoT Python SDK logging")
    return parser

# An MQTT shadow client that uploads device performance data to AWS IoT at a regular interval.
class PerformanceShadowClient:
    def __init__(self, thingName, host, port, rootCAPath, privateKeyPath, certificatePath, requestDelay):
        self.thingName = thingName
        self.host = host
        self.port = port
        self.rootCAPath = rootCAPath
        self.privateKeyPath = privateKeyPath
        self.certificatePath = certificatePath
        self.requestDelay = requestDelay

    # Updates this thing's shadow with system performance data at a regular interval.
    def run(self):
        print("Connecting MQTT client for {}...".format(self.thingName))
        mqttClient = self.configureMQTTClient()
        mqttClient.connect()
        print("MQTT client for {} connected".format(self.thingName))
        deviceShadowHandler = mqttClient.createShadowHandlerWithName(self.thingName, True)

        print("Running performance shadow client for {}...\n".format(self.thingName))
        while True:
            performance = self.readPerformance()
            print("[{}]".format(self.thingName))
            print("CPU:\t{}%".format(performance["cpu"]))
            print("Memory:\t{}%\n".format(performance["memory"]))
            payload = { "state": { "reported": performance } }
            deviceShadowHandler.shadowUpdate(json.dumps(payload), self.shadowUpdateCallback, 5)
            time.sleep(args.requestDelay)

    # Configures the MQTT shadow client for this thing.
    def configureMQTTClient(self):
        mqttClient = AWSIoTMQTTShadowClient(self.thingName)
        mqttClient.configureEndpoint(self.host, self.port)
        mqttClient.configureCredentials(self.rootCAPath, self.privateKeyPath, self.certificatePath)
        mqttClient.configureAutoReconnectBackoffTime(1, 32, 20)
        mqttClient.configureConnectDisconnectTimeout(10)
        mqttClient.configureMQTTOperationTimeout(5)
        return mqttClient

    # Returns the local device's CPU usage, memory usage, and timestamp.
    def readPerformance(self):
        cpu = psutil.cpu_percent()
        memory = psutil.virtual_memory().percent
        timestamp = time.time()
        return { "cpu": cpu, "memory": memory, "timestamp": timestamp }

    # Prints the result of a shadow update call.
    def shadowUpdateCallback(self, payload, responseStatus, token):
        print("[{}]".format(self.thingName))
        print("Update request {} {}\n".format(token, responseStatus))

# Configures debug logging for the AWS IoT Python SDK.
def configureLogging():
    logger = logging.getLogger("AWSIoTPythonSDK.core")
    logger.setLevel(logging.DEBUG)
    streamHandler = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    streamHandler.setFormatter(formatter)
    logger.addHandler(streamHandler)

# Runs the performance shadow client with user arguments.
if __name__ == "__main__":
    parser = configureParser()
    args = parser.parse_args()
    if (args.enableLogging):
        configureLogging()
    thingClient = PerformanceShadowClient(args.thingName, args.host, args.port, args.rootCAPath, args.privateKeyPath,
            args.certificatePath, args.requestDelay)
    thingClient.run()

# snippet-end:[iot.python.thing_performance.complete]
<commit_msg>Edit for comments in thing_performance.py<commit_after># Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.

# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

# snippet-comment:[These are tags for the AWS doc team's sample catalog. Do not remove.]
# snippet-sourcedescription:[thing_performance.py demonstrates how to push CPU and memory usage data to a thing's device shadow in AWS IoT.]
# snippet-service:[iot]
# snippet-sourcesyntax:[python]
# snippet-keyword:[Python]
# snippet-keyword:[AWS IoT]
# snippet-keyword:[Code Sample]
# snippet-keyword:[AWSIoTSDK]
# snippet-keyword:[AWSIoTMQTTShadowClient]
# snippet-sourcetype:[full-example]
# snippet-sourcedate:[2020-01-23]
# snippet-sourceauthor:[FThompsonAWS]
# snippet-start:[iot.python.thing_performance.complete]

from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTShadowClient

import json
import psutil
import argparse
import logging
import time

def configureParser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-e", "--endpoint", action="store", required=True, dest="host",
            help="Your AWS IoT custom endpoint")
    parser.add_argument("-r", "--rootCA", action="store", required=True, dest="rootCAPath", help="Root CA file path")
    parser.add_argument("-c", "--cert", action="store", required=True, dest="certificatePath",
            help="Certificate file path")
    parser.add_argument("-k", "--key", action="store", required=True, dest="privateKeyPath",
            help="Private key file path")
    parser.add_argument("-p", "--port", action="store", dest="port", type=int, default=8883,
            help="Port number override")
    parser.add_argument("-n", "--thingName", action="store", required=True, dest="thingName",
            help="Targeted thing name")
    parser.add_argument("-d", "--requestDelay", action="store", dest="requestDelay", type=float, default=1,
            help="Time between requests (in seconds)")
    parser.add_argument("-v", "--enableLogging", action="store_true", dest="enableLogging",
            help="Enable logging for the AWS IoT Device SDK for Python")
    return parser

# An MQTT shadow client that uploads device performance data to AWS IoT at a regular interval.
class PerformanceShadowClient:
    def __init__(self, thingName, host, port, rootCAPath, privateKeyPath, certificatePath, requestDelay):
        self.thingName = thingName
        self.host = host
        self.port = port
        self.rootCAPath = rootCAPath
        self.privateKeyPath = privateKeyPath
        self.certificatePath = certificatePath
        self.requestDelay = requestDelay

    # Updates this thing's shadow with system performance data at a regular interval.
    def run(self):
        print("Connecting MQTT client for {}...".format(self.thingName))
        mqttClient = self.configureMQTTClient()
        mqttClient.connect()
        print("MQTT client for {} connected".format(self.thingName))
        deviceShadowHandler = mqttClient.createShadowHandlerWithName(self.thingName, True)

        print("Running performance shadow client for {}...\n".format(self.thingName))
        while True:
            performance = self.readPerformance()
            print("[{}]".format(self.thingName))
            print("CPU:\t{}%".format(performance["cpu"]))
            print("Memory:\t{}%\n".format(performance["memory"]))
            payload = { "state": { "reported": performance } }
            deviceShadowHandler.shadowUpdate(json.dumps(payload), self.shadowUpdateCallback, 5)
            time.sleep(args.requestDelay)

    # Configures the MQTT shadow client for this thing.
    def configureMQTTClient(self):
        mqttClient = AWSIoTMQTTShadowClient(self.thingName)
        mqttClient.configureEndpoint(self.host, self.port)
        mqttClient.configureCredentials(self.rootCAPath, self.privateKeyPath, self.certificatePath)
        mqttClient.configureAutoReconnectBackoffTime(1, 32, 20)
        mqttClient.configureConnectDisconnectTimeout(10)
        mqttClient.configureMQTTOperationTimeout(5)
        return mqttClient

    # Returns the local device's CPU usage, memory usage, and timestamp.
    def readPerformance(self):
        cpu = psutil.cpu_percent()
        memory = psutil.virtual_memory().percent
        timestamp = time.time()
        return { "cpu": cpu, "memory": memory, "timestamp": timestamp }

    # Prints the result of a shadow update call.
    def shadowUpdateCallback(self, payload, responseStatus, token):
        print("[{}]".format(self.thingName))
        print("Update request {} {}\n".format(token, responseStatus))

# Configures debug logging for the AWS IoT Device SDK for Python.
def configureLogging():
    logger = logging.getLogger("AWSIoTPythonSDK.core")
    logger.setLevel(logging.DEBUG)
    streamHandler = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    streamHandler.setFormatter(formatter)
    logger.addHandler(streamHandler)

# Runs the performance shadow client with user arguments.
if __name__ == "__main__":
    parser = configureParser()
    args = parser.parse_args()
    if (args.enableLogging):
        configureLogging()
    thingClient = PerformanceShadowClient(args.thingName, args.host, args.port, args.rootCAPath, args.privateKeyPath,
            args.certificatePath, args.requestDelay)
    thingClient.run()

# snippet-end:[iot.python.thing_performance.complete]
<|endoftext|>


<commit_before># Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from u2flib_server.attestation.metadata import MetadataProvider
from u2flib_server.attestation.resolvers import create_resolver
from u2flib_server.attestation.data import YUBICO
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from base64 import b64decode
import json
import unittest

ATTESTATION_CERT = b64decode(b"""
MIICGzCCAQWgAwIBAgIEdaP2dTALBgkqhkiG9w0BAQswLjEsMCoGA1UEAxMjWXViaWNvIFUyRiBS
b290IENBIFNlcmlhbCA0NTcyMDA2MzEwIBcNMTQwODAxMDAwMDAwWhgPMjA1MDA5MDQwMDAwMDBa
MCoxKDAmBgNVBAMMH1l1YmljbyBVMkYgRUUgU2VyaWFsIDE5NzM2Nzk3MzMwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAAQZo35Damtpl81YdmcbhEuXKAr7xDcQzAy5n3ftAAhtBbu8EeGU4ynfSgLo
nckqX6J2uXLBppTNE3v2bt+Yf8MLoxIwEDAOBgorBgEEAYLECgECBAAwCwYJKoZIhvcNAQELA4IB
AQC9LbiNPgs0sQYOHAJcg+lMk+HCsiWRlYVnbT4I/5lnqU907vY17XYAORd432bU3Nnhsbkvjz76
kQJGXeNAF4DPANGGlz8JU+LNEVE2PWPGgEM0GXgB7mZN5Sinfy1AoOdO+3c3bfdJQuXlUxHbo+nD
pxxKpzq9gr++RbokF1+0JBkMbaA/qLYL4WdhY5NvaOyMvYpO3sBxlzn6FcP67hlotGH1wU7qhCeh
+uur7zDeAWVh7c4QtJOXHkLJQfV3Z7ZMvhkIA6jZJAX99hisABU/SSa5DtgX7AfsHwa04h69AAAW
DUzSk3HgOXbUd1FaSOPdlVFkG2N2JllFHykyO3zO
""")

class AttestationTest(unittest.TestCase):

    def test_resolver(self):
        resolver = create_resolver(YUBICO)
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT, default_backend())
        metadata = resolver.resolve(cert)
        assert metadata.identifier == '2fb54029-7613-4f1d-94f1-fb876c14a6fe'

    def test_provider(self):
        provider = MetadataProvider()
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT, default_backend())
        attestation = provider.get_attestation(cert)

        assert attestation.trusted

    def test_versioning_newer(self):
        resolver = create_resolver(YUBICO)
        newer = json.loads(json.dumps(YUBICO))
        newer['version'] = newer['version'] + 1
        newer['trustedCertificates'] = []

        resolver.add_metadata(newer)

        cert = x509.load_der_x509_certificate(ATTESTATION_CERT, default_backend())
        metadata = resolver.resolve(cert)

        assert metadata is None

    def test_versioning_older(self):
        resolver = create_resolver(YUBICO)
        newer = json.loads(json.dumps(YUBICO))
        newer['trustedCertificates'] = []

        resolver.add_metadata(newer)

        cert = x509.load_der_x509_certificate(ATTESTATION_CERT, default_backend())
        metadata = resolver.resolve(cert)

        assert metadata.identifier == '2fb54029-7613-4f1d-94f1-fb876c14a6fe'
<commit_msg>Added tests for reading transport data.<commit_after># Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from u2flib_server.attestation.metadata import MetadataProvider, Transport
from u2flib_server.attestation.resolvers import create_resolver
from u2flib_server.attestation.data import YUBICO
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from base64 import b64decode
import json
import unittest

ATTESTATION_CERT = b64decode(b"""
MIICGzCCAQWgAwIBAgIEdaP2dTALBgkqhkiG9w0BAQswLjEsMCoGA1UEAxMjWXViaWNvIFUyRiBS
b290IENBIFNlcmlhbCA0NTcyMDA2MzEwIBcNMTQwODAxMDAwMDAwWhgPMjA1MDA5MDQwMDAwMDBa
MCoxKDAmBgNVBAMMH1l1YmljbyBVMkYgRUUgU2VyaWFsIDE5NzM2Nzk3MzMwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAAQZo35Damtpl81YdmcbhEuXKAr7xDcQzAy5n3ftAAhtBbu8EeGU4ynfSgLo
nckqX6J2uXLBppTNE3v2bt+Yf8MLoxIwEDAOBgorBgEEAYLECgECBAAwCwYJKoZIhvcNAQELA4IB
AQC9LbiNPgs0sQYOHAJcg+lMk+HCsiWRlYVnbT4I/5lnqU907vY17XYAORd432bU3Nnhsbkvjz76
kQJGXeNAF4DPANGGlz8JU+LNEVE2PWPGgEM0GXgB7mZN5Sinfy1AoOdO+3c3bfdJQuXlUxHbo+nD
pxxKpzq9gr++RbokF1+0JBkMbaA/qLYL4WdhY5NvaOyMvYpO3sBxlzn6FcP67hlotGH1wU7qhCeh
+uur7zDeAWVh7c4QtJOXHkLJQfV3Z7ZMvhkIA6jZJAX99hisABU/SSa5DtgX7AfsHwa04h69AAAW
DUzSk3HgOXbUd1FaSOPdlVFkG2N2JllFHykyO3zO
""")

ATTESTATION_CERT_WITH_TRANSPORT = b64decode(b"""
MIICIjCCAQygAwIBAgIEIHHwozALBgkqhkiG9w0BAQswDzENMAsGA1UEAxMEdGVz
dDAeFw0xNTA4MTEwOTAwMzNaFw0xNjA4MTAwOTAwMzNaMCkxJzAlBgNVBAMTHll1
YmljbyBVMkYgRUUgU2VyaWFsIDU0NDMzODA4MzBZMBMGByqGSM49AgEGCCqGSM49
AwEHA0IABPdFG1pBjBBQVhLrD39Qg1vKjuR2kRdBZnwLI/zgzztQpf4ffpkrkB/3
E0TXj5zg8gN9sgMkX48geBe+tBEpvMmjOzA5MCIGCSsGAQQBgsQKAgQVMS4zLjYu
MS40LjEuNDE0ODIuMS4yMBMGCysGAQQBguUcAgEBBAQDAgQwMAsGCSqGSIb3DQEB
CwOCAQEAb3YpnmHHduNuWEXlLqlnww9034ZeZaojhPAYSLR8d5NPk9gc0hkjQKmI
aaBM7DsaHbcHMKpXoMGTQSC++NCZTcKvZ0Lt12mp5HRnM1NNBPol8Hte5fLmvW4t
Q9EzLl4gkz7LSlORxTuwTbae1eQqNdxdeB+0ilMFCEUc+3NGCNM0RWd+sP5+gzMX
BDQAI1Sc9XaPIg8t3du5JChAl1ifpu/uERZ2WQgtxeBDO6z1Xoa5qz4svf5oURjP
ZjxS0WUKht48Z2rIjk5lZzERSaY3RrX3UtrnZEIzCmInXOrcRPeAD4ZutpiwuHe6
2ABsjuMRnKbATbOUiLdknNyPYYQz2g==
""")

YUBICO_RESOLVER = create_resolver(YUBICO)
EMPTY_RESOLVER = create_resolver([])

class AttestationTest(unittest.TestCase):

    def test_resolver(self):
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT,
                                              default_backend())
        metadata = YUBICO_RESOLVER.resolve(cert)
        assert metadata.identifier == '2fb54029-7613-4f1d-94f1-fb876c14a6fe'

    def test_provider(self):
        provider = MetadataProvider(YUBICO_RESOLVER)
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT,
                                              default_backend())
        attestation = provider.get_attestation(cert)

        assert attestation.trusted

    def test_versioning_newer(self):
        resolver = create_resolver(YUBICO)
        newer = json.loads(json.dumps(YUBICO))
        newer['version'] = newer['version'] + 1
        newer['trustedCertificates'] = []

        resolver.add_metadata(newer)

        cert = x509.load_der_x509_certificate(ATTESTATION_CERT,
                                              default_backend())
        metadata = resolver.resolve(cert)

        assert metadata is None

    def test_versioning_older(self):
        resolver = create_resolver(YUBICO)
        newer = json.loads(json.dumps(YUBICO))
        newer['trustedCertificates'] = []

        resolver.add_metadata(newer)

        cert = x509.load_der_x509_certificate(ATTESTATION_CERT,
                                              default_backend())
        metadata = resolver.resolve(cert)

        assert metadata.identifier == '2fb54029-7613-4f1d-94f1-fb876c14a6fe'

    def test_transports_from_cert(self):
        provider = MetadataProvider(EMPTY_RESOLVER)
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT_WITH_TRANSPORT,
                                              default_backend())
        attestation = provider.get_attestation(cert)

        assert attestation.transports == Transport.USB | Transport.NFC

    def test_transports_from_metadata(self):
        provider = MetadataProvider(YUBICO_RESOLVER)
        cert = x509.load_der_x509_certificate(ATTESTATION_CERT,
                                              default_backend())
        attestation = provider.get_attestation(cert)
        assert attestation.transports == Transport.USB
<|endoftext|>


<commit_before>'''
Created on 6 jan. 2013

@author: sander
'''
from bitstring import ConstBitStream, BitArray, Bits
from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
from pylisp.utils.afi import read_afi_address_from_bitstream, \
    get_bitstream_for_afi_address
from pylisp.utils.lcaf.base import LCAFAddress
from pylisp.utils.represent import represent
import numbers

__all__ = ['LocatorRecord']

class LocatorRecord(object):
    def __init__(self, priority=255, weight=0, m_priority=255, m_weight=0,
                 local=False, probed_locator=False, reachable=False,
                 locator=None):
        '''
        Constructor
        '''
        # Set defaults
        self.priority = priority
        self.weight = weight
        self.m_priority = m_priority
        self.m_weight = m_weight
        self.local = local
        self.probed_locator = probed_locator
        self.reachable = reachable
        self.locator = locator

        # Store space for reserved bits
        self._reserved1 = BitArray(13)

    def __repr__(self):
        return represent(self.__class__.__name__, self.__dict__)

    def sanitize(self):
        '''
        Check if the current settings conform to the LISP specifications and
        fix where possible.
        '''
        # Priority:  each RLOC is assigned a unicast priority.  Lower values
        # are more preferable.  When multiple RLOCs have the same priority,
        # they MAY be used in a load-split fashion.  A value of 255 means
        # the RLOC MUST NOT be used for unicast forwarding.
        if not isinstance(self.priority, numbers.Integral) \
        or self.priority < 0 or self.priority > 255:
            raise ValueError('Invalid priority')

        # Weight:  when priorities are the same for multiple RLOCs, the weight
        # indicates how to balance unicast traffic between them.  Weight is
        # encoded as a relative weight of total unicast packets that match
        # the mapping entry.  For example if there are 4 locators in a
        # locator set, where the weights assigned are 30, 20, 20, and 10,
        # the first locator will get 37.5% of the traffic, the 2nd and 3rd
        # locators will get 25% of traffic and the 4th locator will get
        # 12.5% of the traffic.  If all weights for a locator-set are equal,
        # receiver of the Map-Reply will decide how to load-split traffic.
        # See Section 6.5 for a suggested hash algorithm to distribute load
        # across locators with same priority and equal weight values.
        #
        # WARNING: Cisco implementations limit the weight to the range 0-100
        if not isinstance(self.weight, numbers.Integral) \
        or self.weight < 0 or self.weight > 255:
            raise ValueError('Invalid weight')

        # M Priority:  each RLOC is assigned a multicast priority used by an
        # ETR in a receiver multicast site to select an ITR in a source
        # multicast site for building multicast distribution trees.  A value
        # of 255 means the RLOC MUST NOT be used for joining a multicast
        # distribution tree.  For more details, see [MLISP].
        if not isinstance(self.m_priority, numbers.Integral) \
        or self.m_priority < 0 or self.m_priority > 255:
            raise ValueError('Invalid multicast priority')

        # M Weight:  when priorities are the same for multiple RLOCs, the
        # weight indicates how to balance building multicast distribution
        # trees across multiple ITRs.  The weight is encoded as a relative
        # weight (similar to the unicast Weights) of total number of trees
        # built to the source site identified by the EID-prefix.  If all
        # weights for a locator-set are equal, the receiver of the Map-Reply
        # will decide how to distribute multicast state across ITRs.  For
        # more details, see [MLISP].
        if not isinstance(self.m_weight, numbers.Integral) \
        or self.m_weight < 0 or self.m_weight > 255:
            raise ValueError('Invalid weight')

        # L: when this bit is set, the locator is flagged as a local locator to
        # the ETR that is sending the Map-Reply.  When a Map-Server is doing
        # proxy Map-Replying [LISP-MS] for a LISP site, the L bit is set to
        # 0 for all locators in this locator-set.
        if not isinstance(self.local, bool):
            raise ValueError('Local flag must be a boolean')

        # p: when this bit is set, an ETR informs the RLOC-probing ITR that the
        # locator address, for which this bit is set, is the one being RLOC-
        # probed and MAY be different from the source address of the Map-
        # Reply.  An ITR that RLOC-probes a particular locator, MUST use
        # this locator for retrieving the data structure used to store the
        # fact that the locator is reachable.  The "p" bit is set for a
        # single locator in the same locator set.  If an implementation sets
        # more than one "p" bit erroneously, the receiver of the Map-Reply
        # MUST select the first locator.  The "p" bit MUST NOT be set for
        # locator-set records sent in Map-Request and Map-Register messages.
        if not isinstance(self.probed_locator, bool):
            raise ValueError('Probed Locator flag must be a boolean')

        # R: set when the sender of a Map-Reply has a route to the locator in
        # the locator data record.  This receiver may find this useful to
        # know if the locator is up but not necessarily reachable from the
        # receiver's point of view.  See also Section 6.4 for another way
        # the R-bit may be used.
        if not isinstance(self.reachable, bool):
            raise ValueError('Reachable flag must be a boolean')

        # Locator:  an IPv4 or IPv6 address (as encoded by the 'Loc-AFI' field)
        # assigned to an ETR.  Note that the destination RLOC address MAY be
        # an anycast address.  A source RLOC can be an anycast address as
        # well.  The source or destination RLOC MUST NOT be the broadcast
        # address (255.255.255.255 or any subnet broadcast address known to
        # the router), and MUST NOT be a link-local multicast address.  The
        # source RLOC MUST NOT be a multicast address.  The destination RLOC
        # SHOULD be a multicast address if it is being mapped from a
        # multicast destination EID.

        if isinstance(self.locator, (IPv4Address, IPv6Address)):
            addresses = [self.locator]
        elif isinstance(self.locator, LCAFAddress):
            addresses = self.locator.get_addresses()
        else:
            raise ValueError('Locator must be an (LCAF) IPv4 or IPv6 address')

        for address in addresses:
            if isinstance(self.locator, IPv4Address):
                if address == IPv4Address(u'255.255.255.255'):
                    raise ValueError('Locator must not be the broadcast '
                                     'address')

                if address in IPv4Network(u'224.0.0.0/24'):
                    raise ValueError('Locator must not be a link-local '
                                     'multicast address')

            elif isinstance(self.locator, IPv6Address):
                if address in IPv6Network(u'ff02::/16') \
                or address in IPv6Network(u'ff12::/16'):
                    raise ValueError('Locator must not be a link-local '
                                     'multicast address')

            else:
                raise ValueError('Locator must be an IPv4 or IPv6 address')

    @classmethod
    def from_bytes(cls, bitstream):
        '''
        Parse the given record and update properties accordingly
        '''
        record = cls()

        # Convert to ConstBitStream (if not already provided)
        if not isinstance(bitstream, ConstBitStream):
            if isinstance(bitstream, Bits):
                bitstream = ConstBitStream(auto=bitstream)
            else:
                bitstream = ConstBitStream(bytes=bitstream)

        # Read the priorities and weights
        (record.priority, record.weight, record.m_priority,
         record.m_weight) = bitstream.readlist('4*uint:8')

        # Read over unused flags
        record._reserved1 = bitstream.read(13)

        # Read the flags
        (record.local,
         record.probed_locator,
         record.reachable) = bitstream.readlist('3*bool')

        # Read the locator
        record.locator = read_afi_address_from_bitstream(bitstream)

        # Verify that the properties make sense
        record.sanitize()

        return record

    def to_bytes(self):
        '''
        Create bytes from properties
        '''
        return self.to_bitstream().bytes

    def to_bitstream(self):
        '''
        Create bitstream from properties
        '''
        # Verify that properties make sense
        self.sanitize()

        # Start with the priorities and weights
        bitstream = BitArray('uint:8=%d, uint:8=%d, uint:8=%d, '
                             'uint:8=%d' % (self.priority,
                                            self.weight,
                                            self.m_priority,
                                            self.m_weight))

        # Add padding
        bitstream += self._reserved1

        # Add the flags
        bitstream += BitArray('bool=%d, bool=%d, bool=%d'
                              % (self.local,
                                 self.probed_locator,
                                 self.reachable))

        # Add the locator
        bitstream += get_bitstream_for_afi_address(self.locator)

        return bitstream
<commit_msg>Rename .locator attribute to .address<commit_after>'''
Created on 6 jan. 2013

@author: sander
'''
from bitstring import ConstBitStream, BitArray, Bits
from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
from pylisp.utils.afi import read_afi_address_from_bitstream, \
    get_bitstream_for_afi_address
from pylisp.utils.lcaf.base import LCAFAddress
from pylisp.utils.represent import represent
import numbers

__all__ = ['LocatorRecord']

class LocatorRecord(object):
    def __init__(self, priority=255, weight=0, m_priority=255, m_weight=0,
                 local=False, probed_locator=False, reachable=False,
                 address=None):
        '''
        Constructor
        '''
        # Set defaults
        self.priority = priority
        self.weight = weight
        self.m_priority = m_priority
        self.m_weight = m_weight
        self.local = local
        self.probed_locator = probed_locator
        self.reachable = reachable
        self.address = address

        # Store space for reserved bits
        self._reserved1 = BitArray(13)

    def __repr__(self):
        return represent(self.__class__.__name__, self.__dict__)

    def sanitize(self):
        '''
        Check if the current settings conform to the LISP specifications and
        fix where possible.
        '''
        # Priority:  each RLOC is assigned a unicast priority.  Lower values
        # are more preferable.  When multiple RLOCs have the same priority,
        # they MAY be used in a load-split fashion.  A value of 255 means
        # the RLOC MUST NOT be used for unicast forwarding.
        if not isinstance(self.priority, numbers.Integral) \
        or self.priority < 0 or self.priority > 255:
            raise ValueError('Invalid priority')

        # Weight:  when priorities are the same for multiple RLOCs, the weight
        # indicates how to balance unicast traffic between them.  Weight is
        # encoded as a relative weight of total unicast packets that match
        # the mapping entry.  For example if there are 4 locators in a
        # locator set, where the weights assigned are 30, 20, 20, and 10,
        # the first locator will get 37.5% of the traffic, the 2nd and 3rd
        # locators will get 25% of traffic and the 4th locator will get
        # 12.5% of the traffic.  If all weights for a locator-set are equal,
        # receiver of the Map-Reply will decide how to load-split traffic.
        # See Section 6.5 for a suggested hash algorithm to distribute load
        # across locators with same priority and equal weight values.
        #
        # WARNING: Cisco implementations limit the weight to the range 0-100
        if not isinstance(self.weight, numbers.Integral) \
        or self.weight < 0 or self.weight > 255:
            raise ValueError('Invalid weight')

        # M Priority:  each RLOC is assigned a multicast priority used by an
        # ETR in a receiver multicast site to select an ITR in a source
        # multicast site for building multicast distribution trees.  A value
        # of 255 means the RLOC MUST NOT be used for joining a multicast
        # distribution tree.  For more details, see [MLISP].
        if not isinstance(self.m_priority, numbers.Integral) \
        or self.m_priority < 0 or self.m_priority > 255:
            raise ValueError('Invalid multicast priority')

        # M Weight:  when priorities are the same for multiple RLOCs, the
        # weight indicates how to balance building multicast distribution
        # trees across multiple ITRs.  The weight is encoded as a relative
        # weight (similar to the unicast Weights) of total number of trees
        # built to the source site identified by the EID-prefix.  If all
        # weights for a locator-set are equal, the receiver of the Map-Reply
        # will decide how to distribute multicast state across ITRs.  For
        # more details, see [MLISP].
        if not isinstance(self.m_weight, numbers.Integral) \
        or self.m_weight < 0 or self.m_weight > 255:
            raise ValueError('Invalid weight')

        # L: when this bit is set, the locator is flagged as a local locator to
        # the ETR that is sending the Map-Reply.  When a Map-Server is doing
        # proxy Map-Replying [LISP-MS] for a LISP site, the L bit is set to
        # 0 for all locators in this locator-set.
        if not isinstance(self.local, bool):
            raise ValueError('Local flag must be a boolean')

        # p: when this bit is set, an ETR informs the RLOC-probing ITR that the
        # locator address, for which this bit is set, is the one being RLOC-
        # probed and MAY be different from the source address of the Map-
        # Reply.  An ITR that RLOC-probes a particular locator, MUST use
        # this locator for retrieving the data structure used to store the
        # fact that the locator is reachable.  The "p" bit is set for a
        # single locator in the same locator set.  If an implementation sets
        # more than one "p" bit erroneously, the receiver of the Map-Reply
        # MUST select the first locator.  The "p" bit MUST NOT be set for
        # locator-set records sent in Map-Request and Map-Register messages.
        if not isinstance(self.probed_locator, bool):
            raise ValueError('Probed Locator flag must be a boolean')

        # R: set when the sender of a Map-Reply has a route to the locator in
        # the locator data record.  This receiver may find this useful to
        # know if the locator is up but not necessarily reachable from the
        # receiver's point of view.  See also Section 6.4 for another way
        # the R-bit may be used.
        if not isinstance(self.reachable, bool):
            raise ValueError('Reachable flag must be a boolean')

        # Locator:  an IPv4 or IPv6 address (as encoded by the 'Loc-AFI' field)
        # assigned to an ETR.  Note that the destination RLOC address MAY be
        # an anycast address.  A source RLOC can be an anycast address as
        # well.  The source or destination RLOC MUST NOT be the broadcast
        # address (255.255.255.255 or any subnet broadcast address known to
        # the router), and MUST NOT be a link-local multicast address.  The
        # source RLOC MUST NOT be a multicast address.  The destination RLOC
        # SHOULD be a multicast address if it is being mapped from a
        # multicast destination EID.

        if isinstance(self.address, (IPv4Address, IPv6Address)):
            addresses = [self.address]
        elif isinstance(self.address, LCAFAddress):
            addresses = self.address.get_addresses()
        else:
            raise ValueError('Locator must be an (LCAF) IPv4 or IPv6 address')

        for address in addresses:
            if isinstance(self.address, IPv4Address):
                if address == IPv4Address(u'255.255.255.255'):
                    raise ValueError('Locator must not be the broadcast '
                                     'address')

                if address in IPv4Network(u'224.0.0.0/24'):
                    raise ValueError('Locator must not be a link-local '
                                     'multicast address')

            elif isinstance(self.address, IPv6Address):
                if address in IPv6Network(u'ff02::/16') \
                or address in IPv6Network(u'ff12::/16'):
                    raise ValueError('Locator must not be a link-local '
                                     'multicast address')

            else:
                raise ValueError('Locator must be an IPv4 or IPv6 address')

    @classmethod
    def from_bytes(cls, bitstream):
        '''
        Parse the given record and update properties accordingly
        '''
        record = cls()

        # Convert to ConstBitStream (if not already provided)
        if not isinstance(bitstream, ConstBitStream):
            if isinstance(bitstream, Bits):
                bitstream = ConstBitStream(auto=bitstream)
            else:
                bitstream = ConstBitStream(bytes=bitstream)

        # Read the priorities and weights
        (record.priority, record.weight, record.m_priority,
         record.m_weight) = bitstream.readlist('4*uint:8')

        # Read over unused flags
        record._reserved1 = bitstream.read(13)

        # Read the flags
        (record.local,
         record.probed_locator,
         record.reachable) = bitstream.readlist('3*bool')

        # Read the locator
        record.address = read_afi_address_from_bitstream(bitstream)

        # Verify that the properties make sense
        record.sanitize()

        return record

    def to_bytes(self):
        '''
        Create bytes from properties
        '''
        return self.to_bitstream().bytes

    def to_bitstream(self):
        '''
        Create bitstream from properties
        '''
        # Verify that properties make sense
        self.sanitize()

        # Start with the priorities and weights
        bitstream = BitArray('uint:8=%d, uint:8=%d, uint:8=%d, '
                             'uint:8=%d' % (self.priority,
                                            self.weight,
                                            self.m_priority,
                                            self.m_weight))

        # Add padding
        bitstream += self._reserved1

        # Add the flags
        bitstream += BitArray('bool=%d, bool=%d, bool=%d'
                              % (self.local,
                                 self.probed_locator,
                                 self.reachable))

        # Add the locator
        bitstream += get_bitstream_for_afi_address(self.address)

        return bitstream
<|endoftext|>


<commit_before><commit_msg>Improve readability path<commit_after><|endoftext|>