<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

import requests
from requests_toolbelt import MultipartEncoder

from upstream.chunk import Chunk
from upstream.exc import FileError, ResponseError, ConnectError

try:
    from urllib import urlretrieve
except ImportError:
    from urllib.request import urlretrieve
try:
    from urllib2 import urlopen, URLError
except ImportError:
    from urllib.retrieve import urlopen, URLError

class Streamer(object):

    def __init__(self, server):
        """ For uploading and downloading files from Metadisk.

        :param server: URL to the Metadisk server
        """
        self.server = server
        self.check_connectivity()

    def check_connectivity(self):
        """
        Check to see if we even get a connection to the server.
        https://stackoverflow.com/questions/3764291/checking-network-connection

        """
        try:
            urlopen(self.server, timeout=1)
        except URLError:
            raise LookupError("Could not connect to server.")

    def download(self, chunk_list, shredder_data = None, destination=""):
        """Download a chunk via GET from the specified node."""

        if len(chunk_list) <= 0:
            pass
        elif len(chunk_list) == 1:
            self._download_chunk(chunk_list[0], destination)
        else:
            for chunk in chunk_list:
                self._download_chunk(chunk, "download/" + chunk.filename)
            shredder_data.merge_chunks()

    def upload(self, path):
        """ Uploads a chunk via POST to the specified node
        to the web-core API.  See API docs:
        https://github.com/Storj/web-core#api-documentation

        :param filepath: Path to file as a string
        :return: upstream.chunk.Chunk
        :raise LookupError: IF
        """
        # Open the file and upload it via POST
        url = self.server + "/api/upload"  # web-core API
        # This is a Transfer-Encoding: chunked filechunk generator
        # that's disabled for fun, also cuz broken APO
        # r = requests.post(url, data=self.filestream(path))
        m = MultipartEncoder(
            {
                'file': ('testfile', open('one-meg.testfile', 'rb'))
            }
        )
        r = requests.post(url, data=m, headers={'Content-Type': m.content_type})

        # Make sure that the API call is actually there
        if r.status_code == 404:
            raise LookupError("API call not found.")
        elif r.status_code == 402:
            raise LookupError("Payment required.")
        elif r.status_code == 500:
            raise LookupError("Server error.")
        elif r.status_code == 201:
            # Everthing checked out, return result
            # based on the format selected
            return Chunk().load_json(r.text)
        else:
            raise LookupError("Received status code %s %s "
                              % (r.status_code, r.reason))

    def _download_chunk(self, chunk, destination=""):
        """ Download a chunk via GET from the specified node.
        https://github.com/storj/web-core

        :param chunk: Information about the chunk to download.
        :param destination: Path where we store the file.
        """

        # Generate request URL
        if chunk.decryptkey == "":
            url = self.server + "/api/download/" + chunk.filehash
        else:
            url = self.server + "/api/download/" + chunk.get_uri()

        # Retrieve chunk from the server and pass it the default file directory
        # or override it to a particular place
        if destination == "":
            return urlretrieve(url, "files/" + chunk.filehash)
        else:
            return urlretrieve(url, destination)

    def filestream(self, filepath):
        """ Streaming file generator

        :param filepath: Path to file to stream
        :raise FileError:
        """
        expandedpath = os.path.expanduser(filepath)
        try:
            os.path.isfile(expandedpath)
        except AssertionError:
            raise FileError("%s not a file or not found" % filepath)

        with open(expandedpath) as f:
            while True:
                chunk = f.read(self.chunk_size)
                if not chunk:
                    break
                yield chunk
<commit_msg>More sensical custom exceptions<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

import requests
from requests_toolbelt import MultipartEncoder

from upstream.chunk import Chunk
from upstream.exc import FileError, ResponseError, ConnectError

try:
    from urllib import urlretrieve
except ImportError:
    from urllib.request import urlretrieve
try:
    from urllib2 import urlopen, URLError
except ImportError:
    from urllib.retrieve import urlopen, URLError

class Streamer(object):

    def __init__(self, server):
        """ For uploading and downloading files from Metadisk.

        :param server: URL to the Metadisk server
        """
        self.server = server
        self.check_connectivity()

    def check_connectivity(self):
        """
        Check to see if we even get a connection to the server.
        https://stackoverflow.com/questions/3764291/checking-network-connection

        """
        try:
            urlopen(self.server, timeout=1)
        except URLError:
            raise ConnectError("Could not connect to server.")

    def download(self, chunk_list, shredder_data = None, destination=""):
        """Download a chunk via GET from the specified node."""

        if len(chunk_list) <= 0:
            pass
        elif len(chunk_list) == 1:
            self._download_chunk(chunk_list[0], destination)
        else:
            for chunk in chunk_list:
                self._download_chunk(chunk, "download/" + chunk.filename)
            shredder_data.merge_chunks()

    def upload(self, path):
        """ Uploads a chunk via POST to the specified node
        to the web-core API.  See API docs:
        https://github.com/Storj/web-core#api-documentation

        :param filepath: Path to file as a string
        :return: upstream.chunk.Chunk
        :raise LookupError: IF
        """
        # Open the file and upload it via POST
        url = self.server + "/api/upload"  # web-core API
        # This is a Transfer-Encoding: chunked filechunk generator
        # that's disabled for fun, also cuz broken APO
        # r = requests.post(url, data=self.filestream(path))
        m = MultipartEncoder(
            {
                'file': ('testfile', open('one-meg.testfile', 'rb'))
            }
        )
        r = requests.post(url, data=m, headers={'Content-Type': m.content_type})

        # Make sure that the API call is actually there
        if r.status_code == 404:
            raise LookupError("API call not found.")
        elif r.status_code == 402:
            raise LookupError("Payment required.")
        elif r.status_code == 500:
            raise LookupError("Server error.")
        elif r.status_code == 201:
            # Everthing checked out, return result
            # based on the format selected
            return Chunk().load_json(r.text)
        else:
            raise LookupError("Received status code %s %s "
                              % (r.status_code, r.reason))

    def _download_chunk(self, chunk, destination=""):
        """ Download a chunk via GET from the specified node.
        https://github.com/storj/web-core

        :param chunk: Information about the chunk to download.
        :param destination: Path where we store the file.
        """

        # Generate request URL
        if chunk.decryptkey == "":
            url = self.server + "/api/download/" + chunk.filehash
        else:
            url = self.server + "/api/download/" + chunk.get_uri()

        # Retrieve chunk from the server and pass it the default file directory
        # or override it to a particular place
        if destination == "":
            return urlretrieve(url, "files/" + chunk.filehash)
        else:
            return urlretrieve(url, destination)

    def _filestream(self, filepath):
        """ Streaming file generator

        :param filepath: Path to file to stream
        :raise FileError: If path is not valid
        """
        expandedpath = os.path.expanduser(filepath)
        try:
            os.path.isfile(expandedpath)
        except AssertionError:
            raise FileError("%s not a file or not found" % filepath)

        with open(expandedpath) as f:
            while True:
                chunk = f.read(self.chunk_size)
                if not chunk:
                    break
                yield chunk
<|endoftext|>


<commit_before>"""
  _    _ ______          _   _______ _    _
 | |  | |  ____|   /\   | | |__   __| |  | |
 | |__| | |__     /  \  | |    | |  | |__| |
 |  __  |  __|   / /\ \ | |    | |  |  __  |
 | |  | | |____ / ____ \| |____| |  | |  | |
 |_|  |_|______/_/    \_\______|_|  |_|  |_|

 - no optional questions - no n/a in cycle1 data
 - one "select all that apply" - visit_reason
   - map valid answers to new columns val True/False
 - remaining columns
   - map known responses to cycle2 values
   - map known responses to 'other' if in cycle2, otherwise n/a
"""

import csv
import pandas as pd

# In alphabetical order of the original column names according to pandas
cycle2_columns = ['waiting_group/medicine_time', 'device_id', 'yes_no_group/examined_private', 'performance_group/respect_admin', 'performance_group/queues', 'performance_group/respect_professionals', 'yes_no_group/consent', 'yes_no_group/safety', 'yes_no_group/all_medication', 'yes_no_group/complaint', 'yes_no_group/complaint_response', 'performance_group/equipment', 'waiting_group/register_time', 'performance_group/ambulance', 'tracking_no', 'personal_comment1', 'personal_comment2', 'clinic_committee', 'clinic_committee_function', 'clinic_feedback', 'improvements_comment', 'demographics_group/gender', 'facility', 'demographics_group/age', 'demographics_group/disability', 'distance', 'demographics_group/income', 'town_village', 'district', 'province', 'surveyor', 'date', 'visit_reason', 'travel_distance', '_uuid', 'waiting_group/professional_time', 'submitted_date', 'performance_group/clean']

# change values

# method
# for val in pd.unique(df.where(df['device_id']=='MAVCEC1')['facility'].ravel()):
#    print val
#
# deviceid doesn't seem to be fixed to a site
#
# df.where(df['town_village']=='Folweni').replace(inplace=True, to_replace={'facility':{'Clinic':'notclinic'}})
# doesn't seem to work

# for c in df.columns:
#     if c.startswith('waiting_group/'):
#         print("### %s ###" % c)
#         for val in pd.unique(df[c].ravel()):
#             print("'%s': ''," % val)

replacements_all = {
    'visit_reason': {
        '3 days for infant': 'accompanying',
        '6 weeks': 'other',
        'Accompanying someone else': 'accompanying',
        'Accompanying someone elseI visit regularly to collect monthly medication and/or check-up.': 'accompanying,regular_collection',
        'Emergency': 'emergency',
        'EmergencyAccompanying someone else': 'emergency,accompanying',
        'EmergencyI visit regularly to collect monthly medication and/or check-up.': 'emergency,accompanying',
        'Family planning': 'other',
        'Generally not feeling well': 'not_well',
        'Generally not feeling wellAccompanying someone else': 'not_well,accompanying',
        'Generally not feeling wellEmergency': 'not_well,emergency',
        'Generally not feeling wellI visit regularly to collect monthly medication and/or check-up.': 'not_well,regular_collection',
        'Generally not feeling wellchild to scale': 'not_well,other',
        'Generally not feeling welltesting': 'not_well,other',
        'I visit regularly to collect monthly medication and/or check-up.': 'regular_collection',
        'Immunisation': 'other',
        'Personal': 'other',
        'Pregnant mother and children': 'pregnant',
        'Pregnant mother and childrenEmergencyI visit regularly to collect monthly medication and/or check-up.': 'pregnant,emergency,regular_collection',
        'baby vaccination': 'other',
        'body height and weight': 'other',
        'checking TB': 'other',
        'checkup': 'other',
        'child check up': 'accompanying',
        'child immunization': 'accompanying',
        'child sick': 'accompanying',
        'child vaccination': 'accompanying',
        'child was not feeling well': 'accompanying',
        'child was sick': 'accompanying',
        'child': 'accompanying',
        'children immunisation': 'accompanying',
        'chronic / high blood': 'other',
        'contraception': 'other',
        'diahria': 'other',
        'diarrhea': 'other',
        'doctor appointment': 'other',
        'family planning': 'other',
        'getting results': 'other',
        'have water bumps under my feet.': 'other',
        'i transferred by hospital': 'other',
        'immunisation': 'other',
        'immunizations': 'other',
        'inLabour': 'pregnant',
        'injection': 'other',
        'maternity': 'pregnant',
        'my children were sick': 'accompanying',
        'new born': 'other',
        'pregnancy test': 'other',
        'pregnancy': 'pregnant',
        'pregnant an to check blood': 'pregnant,other',
        'preventing': 'other',
        'prevention': 'other',
        'private matter': 'other',
        'private': 'other',
        'taking child to the clinic': 'accompanying',
        'test Tb': 'other',
        'testing HIV and high blood': 'other',
        'testing': 'other',
        'the child is sick': 'accompanying',
        'to check BP and diabetes.': 'other',
        'to check BP': 'other',
        'to check bp': 'other',
        'tooth pains': 'other',
        'transfer': 'other',
        'treatment': 'other',
        'vommiting and drousy': 'other',
    },
    'travel_distance': {
        '2 km or less': 'under_two',
        'More than 8 km': 'more_eight',
        '3 - 5 km': 'two_five',
        '6 - 8 km': 'six_eight',
    },
    'waiting_group/register_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        'More than 4 hours3 - 4 hours': 'n/a',
        'More than 4 hoursLess than 1 hour': 'n/a',
    },
    'waiting_group/professional_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        '2 - 3 hours1 - 2 hours': 'n/a',
        '3 - 4 hours2 - 3 hours1 - 2 hours': 'n/a',
        '2 - 3 hoursLess than 1 hour': 'n/a',
    },
    'waiting_group/medicine_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        'More than 4 hours3 - 4 hours': 'n/a',
        'More than 4 hoursLess than 1 hour': 'n/a',
    },
    'yes_no_group/safety': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/examined_private': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/consent': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/all_medication': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/complaint': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/complaint_response': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'distance': {
        'Yes': 'yes',
        'No': 'no',
    },
    'performance_group/clean': {
        'Very poor': 1,
        'Poor': 2,
        'Not good.not bad': 3,
        'Good': 4,
        'Excellent': 5,
        'Not good.not badGood': 'n/a',
        'PoorGood': 'n/a',
    },
    'performance_group/queues': {
        'Excellent': 5,
        'Good': 4,
        'GoodExcellent': 'n/a',
        'Not good.not bad': 3,
        'Poor': 2,
        'PoorGood': 'n/a',
        'Very poor': 1,
        'Very poorExcellent': 'n/a',
        'Very poorGood': 'n/a',
        'Very poorPoor': 'n/a',
    },
    'performance_group/respect_admin': {
        'Excellent': 5,
        'Good': 4,
        'Not good.not bad': 3,
        'Not good.not badExcellent': 'n/a',
        'Poor': 2,
        'PoorGood': 'n/a',
        'Very poor': 1,
        'Very poorGood': 'n/a',
    },
    'performance_group/respect_professionals': {
        'Excellent': 5,
        'Good': 4,
        'GoodExcellent': 'n/a',
        'Not good.not bad': 3,
        'Not good.not badExcellent': 'n/a',
        'Poor': 2,
        'Very poor': 1,
        'Very poorExcellent': 'n/a',
        'Very poorGood': 'n/a',
        'Very poorNot good.not bad': 'n/a',
    },
    'performance_group/ambulance': {
        'Excellent': 5,
        'Good': 4,
        'Not good.not bad': 3,
        'Not good.not badExcellent': 'n/a',
        'Poor': 2,
        'PoorGood': 'n/a',
        'Very poor': 1,
        'Very poorExcellent': 'n/a',
        'Very poorPoorExcellent': 'n/a',
    },
    'performance_group/equipment': {
        'Excellent': 5,
        'Good': 4,
        'Not good.not bad': 3,
        'Poor': 2,
        'PoorExcellent': 'n/a',
        'Very poor': 1,
        'Very poorExcellent': 'n/a',
    },
    'clinic_committee': {
        'Yes': 'yes',
        'No': 'no',
        'Don\'t know': 'unsure',
    },
    'clinic_committee_function': {
        'Yes': 'yes',
        'No': 'no',
        'Don\'t know': 'n/a',
    },
    'demographics_group/gender': {
        'Male': 'male',
        'Female': 'female',
    },
    'demographics_group/age': {
        '19 - 40 years old': '19_40',
        '41 - 60 years old': '40_60',
        'Older than 60 years': 'older_60',
        'Under 18 years old': 'under_18',
    },
    'demographics_group/disability': {
        'No': 'no',
        'Yes': 'yes',
    },
    'demographics_group/income': {
        'Government grant': 'gov_grant',
        'I do not earn income': 'none',
        'Permanent employment': 'perm_employ',
        'Temporary employment': 'temp_employ',
        'Small scale vendor (no employees)': 'small_vendor',
        'Own business (with employees)': 'own_business',
    },
}

device_replacements = {
    'MAVCEC1': {
        'facility': {
            'Thabong Clinic': 'thabong',
            'Thabong': 'thabong',
        },
    },
    'MAVCEC4': {
        'facility': {
            'Port St Johns': 'port_st_johns',
        },
    },
    'MAVCJHB5': {
        'facility': {
            'Soshoanguve Block X Clinic': 'tshwane',
            'Clinic': 'tshwane',
            'ufvh': 'tshwane',
            'ju5huh': 'tshwane',
            'Soshoanguve': 'tshwane',
        },
    },
    'MAVCKZN2': {
        'facility': {
            'siphosethu nzoyi': 'umlazi',
            'Q clinic': 'umlazi',
            'D': 'umlazi',
        },
    },
    'MAVCKZN5': {
        'facility': {
            'Clinic': 'folweni',
        },
    },
}

device_files = {
    'MAVCEC1': 'Health Citizen Survey MAVCEC1 - Data.csv',
    'MAVCEC4': 'Health Citizen Survey MAVCEC4 - Data.csv',
    'MAVCJHB5': 'Health Citizen Survey MAVCJHB5 - Data.csv',
    'MAVCKZN2': 'Health Citizen Survey MAVCKZN2 - Data.csv',
    'MAVCKZN5': 'Health Citizen Survey MAVCKZN5 - Data.csv'
}

# [c for c in df2.columns if c.startswith("visit_reason")]
visit_reason_vals = ['accompanying',
                     'emergency',
                     'not_well',
                     'other',
                     'pregnant',
                     'regular_collection']

def set_select_all_that_apply_columns(dict, q_key, possible_vals):
    for val in possible_vals:
        dict['/'.join([q_key, val])] = 'False'
    for val in dict[q_key].split(','):
        dict['/'.join([q_key, val])] = 'True'
    return dict

def run():
    dicts = []

    # Read in each device file and do the per-device replacements
    for id, filename in device_files.iteritems():
        f = open('cycle1/' + filename)
        r = csv.DictReader(f, delimiter=',', quotechar='"')
        df = pd.DataFrame(list(r))
        df.columns = cycle2_columns
        df.replace(to_replace=device_replacements[id], inplace=True)
        dicts += df.T.to_dict().values()

    df = pd.DataFrame(dicts)
    df.replace(to_replace=replacements_all, inplace=True)
    dicts = df.T.to_dict().values()
    dicts = map(
        lambda x: set_select_all_that_apply_columns(x, 'visit_reason', visit_reason_vals),
        dicts
    )
    return dicts
<commit_msg>fix numeric key bug - ints as strings<commit_after>"""
  _    _ ______          _   _______ _    _
 | |  | |  ____|   /\   | | |__   __| |  | |
 | |__| | |__     /  \  | |    | |  | |__| |
 |  __  |  __|   / /\ \ | |    | |  |  __  |
 | |  | | |____ / ____ \| |____| |  | |  | |
 |_|  |_|______/_/    \_\______|_|  |_|  |_|

 - no optional questions - no n/a in cycle1 data
 - one "select all that apply" - visit_reason
   - map valid answers to new columns val True/False
 - remaining columns
   - map known responses to cycle2 values
   - map known responses to 'other' if in cycle2, otherwise n/a
"""

import csv
import pandas as pd

# In alphabetical order of the original column names according to pandas
cycle2_columns = ['waiting_group/medicine_time', 'device_id', 'yes_no_group/examined_private', 'performance_group/respect_admin', 'performance_group/queues', 'performance_group/respect_professionals', 'yes_no_group/consent', 'yes_no_group/safety', 'yes_no_group/all_medication', 'yes_no_group/complaint', 'yes_no_group/complaint_response', 'performance_group/equipment', 'waiting_group/register_time', 'performance_group/ambulance', 'tracking_no', 'personal_comment1', 'personal_comment2', 'clinic_committee', 'clinic_committee_function', 'clinic_feedback', 'improvements_comment', 'demographics_group/gender', 'facility', 'demographics_group/age', 'demographics_group/disability', 'distance', 'demographics_group/income', 'town_village', 'district', 'province', 'surveyor', 'date', 'visit_reason', 'travel_distance', '_uuid', 'waiting_group/professional_time', 'submitted_date', 'performance_group/clean']

# change values

# method
# for val in pd.unique(df.where(df['device_id']=='MAVCEC1')['facility'].ravel()):
#    print val
#
# deviceid doesn't seem to be fixed to a site
#
# df.where(df['town_village']=='Folweni').replace(inplace=True, to_replace={'facility':{'Clinic':'notclinic'}})
# doesn't seem to work

# for c in df.columns:
#     if c.startswith('waiting_group/'):
#         print("### %s ###" % c)
#         for val in pd.unique(df[c].ravel()):
#             print("'%s': ''," % val)

replacements_all = {
    'visit_reason': {
        '3 days for infant': 'accompanying',
        '6 weeks': 'other',
        'Accompanying someone else': 'accompanying',
        'Accompanying someone elseI visit regularly to collect monthly medication and/or check-up.': 'accompanying,regular_collection',
        'Emergency': 'emergency',
        'EmergencyAccompanying someone else': 'emergency,accompanying',
        'EmergencyI visit regularly to collect monthly medication and/or check-up.': 'emergency,accompanying',
        'Family planning': 'other',
        'Generally not feeling well': 'not_well',
        'Generally not feeling wellAccompanying someone else': 'not_well,accompanying',
        'Generally not feeling wellEmergency': 'not_well,emergency',
        'Generally not feeling wellI visit regularly to collect monthly medication and/or check-up.': 'not_well,regular_collection',
        'Generally not feeling wellchild to scale': 'not_well,other',
        'Generally not feeling welltesting': 'not_well,other',
        'I visit regularly to collect monthly medication and/or check-up.': 'regular_collection',
        'Immunisation': 'other',
        'Personal': 'other',
        'Pregnant mother and children': 'pregnant',
        'Pregnant mother and childrenEmergencyI visit regularly to collect monthly medication and/or check-up.': 'pregnant,emergency,regular_collection',
        'baby vaccination': 'other',
        'body height and weight': 'other',
        'checking TB': 'other',
        'checkup': 'other',
        'child check up': 'accompanying',
        'child immunization': 'accompanying',
        'child sick': 'accompanying',
        'child vaccination': 'accompanying',
        'child was not feeling well': 'accompanying',
        'child was sick': 'accompanying',
        'child': 'accompanying',
        'children immunisation': 'accompanying',
        'chronic / high blood': 'other',
        'contraception': 'other',
        'diahria': 'other',
        'diarrhea': 'other',
        'doctor appointment': 'other',
        'family planning': 'other',
        'getting results': 'other',
        'have water bumps under my feet.': 'other',
        'i transferred by hospital': 'other',
        'immunisation': 'other',
        'immunizations': 'other',
        'inLabour': 'pregnant',
        'injection': 'other',
        'maternity': 'pregnant',
        'my children were sick': 'accompanying',
        'new born': 'other',
        'pregnancy test': 'other',
        'pregnancy': 'pregnant',
        'pregnant an to check blood': 'pregnant,other',
        'preventing': 'other',
        'prevention': 'other',
        'private matter': 'other',
        'private': 'other',
        'taking child to the clinic': 'accompanying',
        'test Tb': 'other',
        'testing HIV and high blood': 'other',
        'testing': 'other',
        'the child is sick': 'accompanying',
        'to check BP and diabetes.': 'other',
        'to check BP': 'other',
        'to check bp': 'other',
        'tooth pains': 'other',
        'transfer': 'other',
        'treatment': 'other',
        'vommiting and drousy': 'other',
    },
    'travel_distance': {
        '2 km or less': 'under_two',
        'More than 8 km': 'more_eight',
        '3 - 5 km': 'two_five',
        '6 - 8 km': 'six_eight',
    },
    'waiting_group/register_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        'More than 4 hours3 - 4 hours': 'n/a',
        'More than 4 hoursLess than 1 hour': 'n/a',
    },
    'waiting_group/professional_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        '2 - 3 hours1 - 2 hours': 'n/a',
        '3 - 4 hours2 - 3 hours1 - 2 hours': 'n/a',
        '2 - 3 hoursLess than 1 hour': 'n/a',
    },
    'waiting_group/medicine_time': {
        'More than 4 hours': 'more_four',
        '1 - 2 hours': 'one_two',
        'Less than 1 hour': 'under_one',
        '2 - 3 hours': 'two_three',
        '3 - 4 hours': 'three_four',
        'More than 4 hours3 - 4 hours': 'n/a',
        'More than 4 hoursLess than 1 hour': 'n/a',
    },
    'yes_no_group/safety': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/examined_private': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/consent': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/all_medication': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/complaint': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'yes_no_group/complaint_response': {
        'Yes': 'yes',
        'No': 'no',
        'YesNo': 'n/a',
    },
    'distance': {
        'Yes': 'yes',
        'No': 'no',
    },
    'performance_group/clean': {
        'Very poor': '1',
        'Poor': '2',
        'Not good.not bad': '3',
        'Good': '4',
        'Excellent': '5',
        'Not good.not badGood': 'n/a',
        'PoorGood': 'n/a',
    },
    'performance_group/queues': {
        'Excellent': '5',
        'Good': '4',
        'GoodExcellent': 'n/a',
        'Not good.not bad': '3',
        'Poor': '2',
        'PoorGood': 'n/a',
        'Very poor': '1',
        'Very poorExcellent': 'n/a',
        'Very poorGood': 'n/a',
        'Very poorPoor': 'n/a',
    },
    'performance_group/respect_admin': {
        'Excellent': '5',
        'Good': '4',
        'Not good.not bad': '3',
        'Not good.not badExcellent': 'n/a',
        'Poor': '2',
        'PoorGood': 'n/a',
        'Very poor': '1',
        'Very poorGood': 'n/a',
    },
    'performance_group/respect_professionals': {
        'Excellent': '5',
        'Good': '4',
        'GoodExcellent': 'n/a',
        'Not good.not bad': '3',
        'Not good.not badExcellent': 'n/a',
        'Poor': '2',
        'Very poor': '1',
        'Very poorExcellent': 'n/a',
        'Very poorGood': 'n/a',
        'Very poorNot good.not bad': 'n/a',
    },
    'performance_group/ambulance': {
        'Excellent': '5',
        'Good': '4',
        'Not good.not bad': '3',
        'Not good.not badExcellent': 'n/a',
        'Poor': '2',
        'PoorGood': 'n/a',
        'Very poor': '1',
        'Very poorExcellent': 'n/a',
        'Very poorPoorExcellent': 'n/a',
    },
    'performance_group/equipment': {
        'Excellent': '5',
        'Good': '4',
        'Not good.not bad': '3',
        'Poor': '2',
        'PoorExcellent': 'n/a',
        'Very poor': '1',
        'Very poorExcellent': 'n/a',
    },
    'clinic_committee': {
        'Yes': 'yes',
        'No': 'no',
        'Don\'t know': 'unsure',
    },
    'clinic_committee_function': {
        'Yes': 'yes',
        'No': 'no',
        'Don\'t know': 'n/a',
    },
    'demographics_group/gender': {
        'Male': 'male',
        'Female': 'female',
    },
    'demographics_group/age': {
        '19 - 40 years old': '19_40',
        '41 - 60 years old': '40_60',
        'Older than 60 years': 'older_60',
        'Under 18 years old': 'under_18',
    },
    'demographics_group/disability': {
        'No': 'no',
        'Yes': 'yes',
    },
    'demographics_group/income': {
        'Government grant': 'gov_grant',
        'I do not earn income': 'none',
        'Permanent employment': 'perm_employ',
        'Temporary employment': 'temp_employ',
        'Small scale vendor (no employees)': 'small_vendor',
        'Own business (with employees)': 'own_business',
    },
}

device_replacements = {
    'MAVCEC1': {
        'facility': {
            'Thabong Clinic': 'thabong',
            'Thabong': 'thabong',
        },
    },
    'MAVCEC4': {
        'facility': {
            'Port St Johns': 'port_st_johns',
        },
    },
    'MAVCJHB5': {
        'facility': {
            'Soshoanguve Block X Clinic': 'tshwane',
            'Clinic': 'tshwane',
            'ufvh': 'tshwane',
            'ju5huh': 'tshwane',
            'Soshoanguve': 'tshwane',
        },
    },
    'MAVCKZN2': {
        'facility': {
            'siphosethu nzoyi': 'umlazi',
            'Q clinic': 'umlazi',
            'D': 'umlazi',
        },
    },
    'MAVCKZN5': {
        'facility': {
            'Clinic': 'folweni',
        },
    },
}

device_files = {
    'MAVCEC1': 'Health Citizen Survey MAVCEC1 - Data.csv',
    'MAVCEC4': 'Health Citizen Survey MAVCEC4 - Data.csv',
    'MAVCJHB5': 'Health Citizen Survey MAVCJHB5 - Data.csv',
    'MAVCKZN2': 'Health Citizen Survey MAVCKZN2 - Data.csv',
    'MAVCKZN5': 'Health Citizen Survey MAVCKZN5 - Data.csv'
}

# [c for c in df2.columns if c.startswith("visit_reason")]
visit_reason_vals = ['accompanying',
                     'emergency',
                     'not_well',
                     'other',
                     'pregnant',
                     'regular_collection']

def set_select_all_that_apply_columns(dict, q_key, possible_vals):
    for val in possible_vals:
        dict['/'.join([q_key, val])] = 'False'
    for val in dict[q_key].split(','):
        dict['/'.join([q_key, val])] = 'True'
    return dict

def run():
    dicts = []

    # Read in each device file and do the per-device replacements
    for id, filename in device_files.iteritems():
        f = open('cycle1/' + filename)
        r = csv.DictReader(f, delimiter=',', quotechar='"')
        df = pd.DataFrame(list(r))
        df.columns = cycle2_columns
        df.replace(to_replace=device_replacements[id], inplace=True)
        dicts += df.T.to_dict().values()

    df = pd.DataFrame(dicts)
    df.replace(to_replace=replacements_all, inplace=True)
    dicts = df.T.to_dict().values()
    dicts = map(
        lambda x: set_select_all_that_apply_columns(x, 'visit_reason', visit_reason_vals),
        dicts
    )
    return dicts
<|endoftext|>


<commit_before>#!/usr/bin/env python

import inspect
import os
import unittest
from framework import VppTestCase
from multiprocessing import Process, Pipe
from pickle import dumps
import six
from six import moves
import sys
from aenum import IntEnum, IntFlag

class SerializableClassCopy(object):
    """
    Empty class used as a basis for a serializable copy of another class.
    """
    pass

    def __repr__(self):
        return '<SerializableClassCopy dict=%s>' % self.__dict__

class RemoteClassAttr(object):
    """
    Wrapper around attribute of a remotely executed class.
    """

    def __init__(self, remote, attr):
        self._path = [attr] if attr else []
        self._remote = remote

    def path_to_str(self):
        return '.'.join(self._path)

    def get_remote_value(self):
        return self._remote._remote_exec(RemoteClass.GET, self.path_to_str())

    def __repr__(self):
        return self._remote._remote_exec(RemoteClass.REPR, self.path_to_str())

    def __str__(self):
        return self._remote._remote_exec(RemoteClass.STR, self.path_to_str())

    def __getattr__(self, attr):
        if attr[0] == '_':
            if not (attr.startswith('__') and attr.endswith('__')):
                raise AttributeError('tried to get private attribute: %s ',
                                     attr)
        self._path.append(attr)
        return self

    def __setattr__(self, attr, val):
        if attr[0] == '_':
            if not (attr.startswith('__') and attr.endswith('__')):
                super(RemoteClassAttr, self).__setattr__(attr, val)
                return
        self._path.append(attr)
        self._remote._remote_exec(RemoteClass.SETATTR, self.path_to_str(),
                                  value=val)

    def __call__(self, *args, **kwargs):
        return self._remote._remote_exec(RemoteClass.CALL, self.path_to_str(),
                                         *args, **kwargs)

class RemoteClass(Process):
    """
    This class can wrap around and adapt the interface of another class,
    and then delegate its execution to a newly forked child process.
    Usage:
        # Create a remotely executed instance of MyClass
        object = RemoteClass(MyClass, arg1='foo', arg2='bar')
        object.start_remote()
        # Access the object normally as if it was an instance of your class.
        object.my_attribute = 20
        print object.my_attribute
        print object.my_method(object.my_attribute)
        object.my_attribute.nested_attribute = 'test'
        # If you need the value of a remote attribute, use .get_remote_value
        method. This method is automatically called when needed in the context
        of a remotely executed class. E.g.:
        if (object.my_attribute.get_remote_value() > 20):
            object.my_attribute2 = object.my_attribute
        # Destroy the instance
        object.quit_remote()
        object.terminate()
    """

    GET = 0       # Get attribute remotely
    CALL = 1      # Call method remotely
    SETATTR = 2   # Set attribute remotely
    REPR = 3      # Get representation of a remote object
    STR = 4       # Get string representation of a remote object
    QUIT = 5      # Quit remote execution

    PIPE_PARENT = 0  # Parent end of the pipe
    PIPE_CHILD = 1  # Child end of the pipe

    DEFAULT_TIMEOUT = 2  # default timeout for an operation to execute

    def __init__(self, cls, *args, **kwargs):
        super(RemoteClass, self).__init__()
        self._cls = cls
        self._args = args
        self._kwargs = kwargs
        self._timeout = RemoteClass.DEFAULT_TIMEOUT
        self._pipe = Pipe()  # pipe for input/output arguments

    def __repr__(self):
        return moves.reprlib.repr(RemoteClassAttr(self, None))

    def __str__(self):
        return str(RemoteClassAttr(self, None))

    def __call__(self, *args, **kwargs):
        return self.RemoteClassAttr(self, None)()

    def __getattr__(self, attr):
        if attr[0] == '_' or not self.is_alive():
            if not (attr.startswith('__') and attr.endswith('__')):
                if hasattr(super(RemoteClass, self), '__getattr__'):
                    return super(RemoteClass, self).__getattr__(attr)
                raise AttributeError('missing: %s', attr)
        return RemoteClassAttr(self, attr)

    def __setattr__(self, attr, val):
        if attr[0] == '_' or not self.is_alive():
            if not (attr.startswith('__') and attr.endswith('__')):
                super(RemoteClass, self).__setattr__(attr, val)
                return
        setattr(RemoteClassAttr(self, None), attr, val)

    def _remote_exec(self, op, path=None, *args, **kwargs):
        """
        Execute given operation on a given, possibly nested, member remotely.
        """
        # automatically resolve remote objects in the arguments
        mutable_args = list(args)
        for i, val in enumerate(mutable_args):
            if isinstance(val, RemoteClass) or \
                    isinstance(val, RemoteClassAttr):
                mutable_args[i] = val.get_remote_value()
        args = tuple(mutable_args)
        for key, val in six.iteritems(kwargs):
            if isinstance(val, RemoteClass) or \
                    isinstance(val, RemoteClassAttr):
                kwargs[key] = val.get_remote_value()
        # send request
        args = self._make_serializable(args)
        kwargs = self._make_serializable(kwargs)
        self._pipe[RemoteClass.PIPE_PARENT].send((op, path, args, kwargs))
        timeout = self._timeout
        # adjust timeout specifically for the .sleep method
        if path is not None and path.split('.')[-1] == 'sleep':
            if args and isinstance(args[0], (long, int)):
                timeout += args[0]
            elif 'timeout' in kwargs:
                timeout += kwargs['timeout']
        if not self._pipe[RemoteClass.PIPE_PARENT].poll(timeout):
            return None
        try:
            rv = self._pipe[RemoteClass.PIPE_PARENT].recv()
            rv = self._deserialize(rv)
            return rv
        except EOFError:
            return None

    def _get_local_object(self, path):
        """
        Follow the path to obtain a reference on the addressed nested attribute
        """
        obj = self._instance
        for attr in path:
            obj = getattr(obj, attr)
        return obj

    def _get_local_value(self, path):
        try:
            return self._get_local_object(path)
        except AttributeError:
            return None

    def _call_local_method(self, path, *args, **kwargs):
        try:
            method = self._get_local_object(path)
            return method(*args, **kwargs)
        except AttributeError:
            return None

    def _set_local_attr(self, path, value):
        try:
            obj = self._get_local_object(path[:-1])
            setattr(obj, path[-1], value)
        except AttributeError:
            pass
        return None

    def _get_local_repr(self, path):
        try:
            obj = self._get_local_object(path)
            return moves.reprlib.repr(obj)
        except AttributeError:
            return None

    def _get_local_str(self, path):
        try:
            obj = self._get_local_object(path)
            return str(obj)
        except AttributeError:
            return None

    def _serializable(self, obj):
        """ Test if the given object is serializable """
        try:
            dumps(obj)
            return True
        except:
            return False

    def _make_obj_serializable(self, obj):
        """
        Make a serializable copy of an object.
        Members which are difficult/impossible to serialize are stripped.
        """
        if self._serializable(obj):
            return obj  # already serializable

        copy = SerializableClassCopy()

        """
        Dictionaries can hold complex values, so we split keys and values into
        separate lists and serialize them individually.
        """
        if (type(obj) is dict):
            copy.type = type(obj)
            copy.k_list = list()
            copy.v_list = list()
            for k, v in obj.items():
                copy.k_list.append(self._make_serializable(k))
                copy.v_list.append(self._make_serializable(v))
            return copy

        # copy at least serializable attributes and properties
        for name, member in inspect.getmembers(obj):
            # skip private members and non-writable dunder methods.
            if name[0] == '_':
                if name in ['__weakref__']:
                    continue
                if name in ['__dict__']:
                    continue
                if not (name.startswith('__') and name.endswith('__')):
                    continue
            if callable(member) and not isinstance(member, property):
                continue
            if not self._serializable(member):
                member = self._make_serializable(member)
            setattr(copy, name, member)
        return copy

    def _make_serializable(self, obj):
        """
        Make a serializable copy of an object or a list/tuple of objects.
        Members which are difficult/impossible to serialize are stripped.
        """
        if (type(obj) is list) or (type(obj) is tuple):
            rv = []
            for item in obj:
                rv.append(self._make_serializable(item))
            if type(obj) is tuple:
                rv = tuple(rv)
            return rv
        elif (isinstance(obj, IntEnum) or isinstance(obj, IntFlag)):
            return obj.value
        else:
            return self._make_obj_serializable(obj)

    def _deserialize_obj(self, obj):
        if (hasattr(obj, 'type')):
            if obj.type is dict:
                _obj = dict()
                for k, v in zip(obj.k_list, obj.v_list):
                    _obj[self._deserialize(k)] = self._deserialize(v)
            return _obj
        return obj

    def _deserialize(self, obj):
        if (type(obj) is list) or (type(obj) is tuple):
            rv = []
            for item in obj:
                rv.append(self._deserialize(item))
            if type(obj) is tuple:
                rv = tuple(rv)
            return rv
        else:
            return self._deserialize_obj(obj)

    def start_remote(self):
        """ Start remote execution """
        self.start()

    def quit_remote(self):
        """ Quit remote execution """
        self._remote_exec(RemoteClass.QUIT, None)

    def get_remote_value(self):
        """ Get value of a remotely held object """
        return RemoteClassAttr(self, None).get_remote_value()

    def set_request_timeout(self, timeout):
        """ Change request timeout """
        self._timeout = timeout

    def run(self):
        """
        Create instance of the wrapped class and execute operations
        on it as requested by the parent process.
        """
        self._instance = self._cls(*self._args, **self._kwargs)
        while True:
            try:
                rv = None
                # get request from the parent process
                (op, path, args,
                 kwargs) = self._pipe[RemoteClass.PIPE_CHILD].recv()
                args = self._deserialize(args)
                kwargs = self._deserialize(kwargs)
                path = path.split('.') if path else []
                if op == RemoteClass.GET:
                    rv = self._get_local_value(path)
                elif op == RemoteClass.CALL:
                    rv = self._call_local_method(path, *args, **kwargs)
                elif op == RemoteClass.SETATTR and 'value' in kwargs:
                    self._set_local_attr(path, kwargs['value'])
                elif op == RemoteClass.REPR:
                    rv = self._get_local_repr(path)
                elif op == RemoteClass.STR:
                    rv = self._get_local_str(path)
                elif op == RemoteClass.QUIT:
                    break
                else:
                    continue
                # send return value
                if not self._serializable(rv):
                    rv = self._make_serializable(rv)
                self._pipe[RemoteClass.PIPE_CHILD].send(rv)
            except EOFError:
                break
        self._instance = None  # destroy the instance

@unittest.skip("Remote Vpp Test Case Class")
class RemoteVppTestCase(VppTestCase):
    """ Re-use VppTestCase to create remote VPP segment

        In your test case:

        @classmethod
        def setUpClass(cls):
            # fork new process before client connects to VPP
            cls.remote_test = RemoteClass(RemoteVppTestCase)

            # start remote process
            cls.remote_test.start_remote()

            # set up your test case
            super(MyTestCase, cls).setUpClass()

            # set up remote test
            cls.remote_test.setUpClass(cls.tempdir)

        @classmethod
        def tearDownClass(cls):
            # tear down remote test
            cls.remote_test.tearDownClass()

            # stop remote process
            cls.remote_test.quit_remote()

            # tear down your test case
            super(MyTestCase, cls).tearDownClass()
    """

    def __init__(self):
        super(RemoteVppTestCase, self).__init__("emptyTest")

    # Note: __del__ is a 'Finalizer" not a 'Destructor'.
    # https://docs.python.org/3/reference/datamodel.html#object.__del__
    def __del__(self):
        if hasattr(self, "vpp"):
            self.vpp.poll()
            if self.vpp.returncode is None:
                self.vpp.terminate()
                self.vpp.communicate()

    @classmethod
    def setUpClass(cls, tempdir):
        # disable features unsupported in remote VPP
        orig_env = dict(os.environ)
        if 'STEP' in os.environ:
            del os.environ['STEP']
        if 'DEBUG' in os.environ:
            del os.environ['DEBUG']
        cls.tempdir_prefix = os.path.basename(tempdir) + "/"
        super(RemoteVppTestCase, cls).setUpClass()
        os.environ = orig_env

    @classmethod
    def tearDownClass(cls):
        super(RemoteVppTestCase, cls).tearDownClass()

    @unittest.skip("Empty test")
    def emptyTest(self):
        """ Do nothing """
        pass

    def setTestFunctionInfo(self, name, doc):
        """
        Store the name and documentation string of currently executed test
        in the main VPP for logging purposes.
        """
        self._testMethodName = name
        self._testMethodDoc = doc
<commit_msg>tests: python3 version check for enum module<commit_after>#!/usr/bin/env python

import inspect
import os
import unittest
from framework import VppTestCase
from multiprocessing import Process, Pipe
from pickle import dumps
import six
from six import moves
import sys

if sys.version_info < (3,):
    from aenum import IntEnum, IntFlag
else:
    from enum import IntEnum, IntFlag

class SerializableClassCopy(object):
    """
    Empty class used as a basis for a serializable copy of another class.
    """
    pass

    def __repr__(self):
        return '<SerializableClassCopy dict=%s>' % self.__dict__

class RemoteClassAttr(object):
    """
    Wrapper around attribute of a remotely executed class.
    """

    def __init__(self, remote, attr):
        self._path = [attr] if attr else []
        self._remote = remote

    def path_to_str(self):
        return '.'.join(self._path)

    def get_remote_value(self):
        return self._remote._remote_exec(RemoteClass.GET, self.path_to_str())

    def __repr__(self):
        return self._remote._remote_exec(RemoteClass.REPR, self.path_to_str())

    def __str__(self):
        return self._remote._remote_exec(RemoteClass.STR, self.path_to_str())

    def __getattr__(self, attr):
        if attr[0] == '_':
            if not (attr.startswith('__') and attr.endswith('__')):
                raise AttributeError('tried to get private attribute: %s ',
                                     attr)
        self._path.append(attr)
        return self

    def __setattr__(self, attr, val):
        if attr[0] == '_':
            if not (attr.startswith('__') and attr.endswith('__')):
                super(RemoteClassAttr, self).__setattr__(attr, val)
                return
        self._path.append(attr)
        self._remote._remote_exec(RemoteClass.SETATTR, self.path_to_str(),
                                  value=val)

    def __call__(self, *args, **kwargs):
        return self._remote._remote_exec(RemoteClass.CALL, self.path_to_str(),
                                         *args, **kwargs)

class RemoteClass(Process):
    """
    This class can wrap around and adapt the interface of another class,
    and then delegate its execution to a newly forked child process.
    Usage:
        # Create a remotely executed instance of MyClass
        object = RemoteClass(MyClass, arg1='foo', arg2='bar')
        object.start_remote()
        # Access the object normally as if it was an instance of your class.
        object.my_attribute = 20
        print object.my_attribute
        print object.my_method(object.my_attribute)
        object.my_attribute.nested_attribute = 'test'
        # If you need the value of a remote attribute, use .get_remote_value
        method. This method is automatically called when needed in the context
        of a remotely executed class. E.g.:
        if (object.my_attribute.get_remote_value() > 20):
            object.my_attribute2 = object.my_attribute
        # Destroy the instance
        object.quit_remote()
        object.terminate()
    """

    GET = 0       # Get attribute remotely
    CALL = 1      # Call method remotely
    SETATTR = 2   # Set attribute remotely
    REPR = 3      # Get representation of a remote object
    STR = 4       # Get string representation of a remote object
    QUIT = 5      # Quit remote execution

    PIPE_PARENT = 0  # Parent end of the pipe
    PIPE_CHILD = 1  # Child end of the pipe

    DEFAULT_TIMEOUT = 2  # default timeout for an operation to execute

    def __init__(self, cls, *args, **kwargs):
        super(RemoteClass, self).__init__()
        self._cls = cls
        self._args = args
        self._kwargs = kwargs
        self._timeout = RemoteClass.DEFAULT_TIMEOUT
        self._pipe = Pipe()  # pipe for input/output arguments

    def __repr__(self):
        return moves.reprlib.repr(RemoteClassAttr(self, None))

    def __str__(self):
        return str(RemoteClassAttr(self, None))

    def __call__(self, *args, **kwargs):
        return self.RemoteClassAttr(self, None)()

    def __getattr__(self, attr):
        if attr[0] == '_' or not self.is_alive():
            if not (attr.startswith('__') and attr.endswith('__')):
                if hasattr(super(RemoteClass, self), '__getattr__'):
                    return super(RemoteClass, self).__getattr__(attr)
                raise AttributeError('missing: %s', attr)
        return RemoteClassAttr(self, attr)

    def __setattr__(self, attr, val):
        if attr[0] == '_' or not self.is_alive():
            if not (attr.startswith('__') and attr.endswith('__')):
                super(RemoteClass, self).__setattr__(attr, val)
                return
        setattr(RemoteClassAttr(self, None), attr, val)

    def _remote_exec(self, op, path=None, *args, **kwargs):
        """
        Execute given operation on a given, possibly nested, member remotely.
        """
        # automatically resolve remote objects in the arguments
        mutable_args = list(args)
        for i, val in enumerate(mutable_args):
            if isinstance(val, RemoteClass) or \
                    isinstance(val, RemoteClassAttr):
                mutable_args[i] = val.get_remote_value()
        args = tuple(mutable_args)
        for key, val in six.iteritems(kwargs):
            if isinstance(val, RemoteClass) or \
                    isinstance(val, RemoteClassAttr):
                kwargs[key] = val.get_remote_value()
        # send request
        args = self._make_serializable(args)
        kwargs = self._make_serializable(kwargs)
        self._pipe[RemoteClass.PIPE_PARENT].send((op, path, args, kwargs))
        timeout = self._timeout
        # adjust timeout specifically for the .sleep method
        if path is not None and path.split('.')[-1] == 'sleep':
            if args and isinstance(args[0], (long, int)):
                timeout += args[0]
            elif 'timeout' in kwargs:
                timeout += kwargs['timeout']
        if not self._pipe[RemoteClass.PIPE_PARENT].poll(timeout):
            return None
        try:
            rv = self._pipe[RemoteClass.PIPE_PARENT].recv()
            rv = self._deserialize(rv)
            return rv
        except EOFError:
            return None

    def _get_local_object(self, path):
        """
        Follow the path to obtain a reference on the addressed nested attribute
        """
        obj = self._instance
        for attr in path:
            obj = getattr(obj, attr)
        return obj

    def _get_local_value(self, path):
        try:
            return self._get_local_object(path)
        except AttributeError:
            return None

    def _call_local_method(self, path, *args, **kwargs):
        try:
            method = self._get_local_object(path)
            return method(*args, **kwargs)
        except AttributeError:
            return None

    def _set_local_attr(self, path, value):
        try:
            obj = self._get_local_object(path[:-1])
            setattr(obj, path[-1], value)
        except AttributeError:
            pass
        return None

    def _get_local_repr(self, path):
        try:
            obj = self._get_local_object(path)
            return moves.reprlib.repr(obj)
        except AttributeError:
            return None

    def _get_local_str(self, path):
        try:
            obj = self._get_local_object(path)
            return str(obj)
        except AttributeError:
            return None

    def _serializable(self, obj):
        """ Test if the given object is serializable """
        try:
            dumps(obj)
            return True
        except:
            return False

    def _make_obj_serializable(self, obj):
        """
        Make a serializable copy of an object.
        Members which are difficult/impossible to serialize are stripped.
        """
        if self._serializable(obj):
            return obj  # already serializable

        copy = SerializableClassCopy()

        """
        Dictionaries can hold complex values, so we split keys and values into
        separate lists and serialize them individually.
        """
        if (type(obj) is dict):
            copy.type = type(obj)
            copy.k_list = list()
            copy.v_list = list()
            for k, v in obj.items():
                copy.k_list.append(self._make_serializable(k))
                copy.v_list.append(self._make_serializable(v))
            return copy

        # copy at least serializable attributes and properties
        for name, member in inspect.getmembers(obj):
            # skip private members and non-writable dunder methods.
            if name[0] == '_':
                if name in ['__weakref__']:
                    continue
                if name in ['__dict__']:
                    continue
                if not (name.startswith('__') and name.endswith('__')):
                    continue
            if callable(member) and not isinstance(member, property):
                continue
            if not self._serializable(member):
                member = self._make_serializable(member)
            setattr(copy, name, member)
        return copy

    def _make_serializable(self, obj):
        """
        Make a serializable copy of an object or a list/tuple of objects.
        Members which are difficult/impossible to serialize are stripped.
        """
        if (type(obj) is list) or (type(obj) is tuple):
            rv = []
            for item in obj:
                rv.append(self._make_serializable(item))
            if type(obj) is tuple:
                rv = tuple(rv)
            return rv
        elif (isinstance(obj, IntEnum) or isinstance(obj, IntFlag)):
            return obj.value
        else:
            return self._make_obj_serializable(obj)

    def _deserialize_obj(self, obj):
        if (hasattr(obj, 'type')):
            if obj.type is dict:
                _obj = dict()
                for k, v in zip(obj.k_list, obj.v_list):
                    _obj[self._deserialize(k)] = self._deserialize(v)
            return _obj
        return obj

    def _deserialize(self, obj):
        if (type(obj) is list) or (type(obj) is tuple):
            rv = []
            for item in obj:
                rv.append(self._deserialize(item))
            if type(obj) is tuple:
                rv = tuple(rv)
            return rv
        else:
            return self._deserialize_obj(obj)

    def start_remote(self):
        """ Start remote execution """
        self.start()

    def quit_remote(self):
        """ Quit remote execution """
        self._remote_exec(RemoteClass.QUIT, None)

    def get_remote_value(self):
        """ Get value of a remotely held object """
        return RemoteClassAttr(self, None).get_remote_value()

    def set_request_timeout(self, timeout):
        """ Change request timeout """
        self._timeout = timeout

    def run(self):
        """
        Create instance of the wrapped class and execute operations
        on it as requested by the parent process.
        """
        self._instance = self._cls(*self._args, **self._kwargs)
        while True:
            try:
                rv = None
                # get request from the parent process
                (op, path, args,
                 kwargs) = self._pipe[RemoteClass.PIPE_CHILD].recv()
                args = self._deserialize(args)
                kwargs = self._deserialize(kwargs)
                path = path.split('.') if path else []
                if op == RemoteClass.GET:
                    rv = self._get_local_value(path)
                elif op == RemoteClass.CALL:
                    rv = self._call_local_method(path, *args, **kwargs)
                elif op == RemoteClass.SETATTR and 'value' in kwargs:
                    self._set_local_attr(path, kwargs['value'])
                elif op == RemoteClass.REPR:
                    rv = self._get_local_repr(path)
                elif op == RemoteClass.STR:
                    rv = self._get_local_str(path)
                elif op == RemoteClass.QUIT:
                    break
                else:
                    continue
                # send return value
                if not self._serializable(rv):
                    rv = self._make_serializable(rv)
                self._pipe[RemoteClass.PIPE_CHILD].send(rv)
            except EOFError:
                break
        self._instance = None  # destroy the instance

@unittest.skip("Remote Vpp Test Case Class")
class RemoteVppTestCase(VppTestCase):
    """ Re-use VppTestCase to create remote VPP segment

        In your test case:

        @classmethod
        def setUpClass(cls):
            # fork new process before client connects to VPP
            cls.remote_test = RemoteClass(RemoteVppTestCase)

            # start remote process
            cls.remote_test.start_remote()

            # set up your test case
            super(MyTestCase, cls).setUpClass()

            # set up remote test
            cls.remote_test.setUpClass(cls.tempdir)

        @classmethod
        def tearDownClass(cls):
            # tear down remote test
            cls.remote_test.tearDownClass()

            # stop remote process
            cls.remote_test.quit_remote()

            # tear down your test case
            super(MyTestCase, cls).tearDownClass()
    """

    def __init__(self):
        super(RemoteVppTestCase, self).__init__("emptyTest")

    # Note: __del__ is a 'Finalizer" not a 'Destructor'.
    # https://docs.python.org/3/reference/datamodel.html#object.__del__
    def __del__(self):
        if hasattr(self, "vpp"):
            self.vpp.poll()
            if self.vpp.returncode is None:
                self.vpp.terminate()
                self.vpp.communicate()

    @classmethod
    def setUpClass(cls, tempdir):
        # disable features unsupported in remote VPP
        orig_env = dict(os.environ)
        if 'STEP' in os.environ:
            del os.environ['STEP']
        if 'DEBUG' in os.environ:
            del os.environ['DEBUG']
        cls.tempdir_prefix = os.path.basename(tempdir) + "/"
        super(RemoteVppTestCase, cls).setUpClass()
        os.environ = orig_env

    @classmethod
    def tearDownClass(cls):
        super(RemoteVppTestCase, cls).tearDownClass()

    @unittest.skip("Empty test")
    def emptyTest(self):
        """ Do nothing """
        pass

    def setTestFunctionInfo(self, name, doc):
        """
        Store the name and documentation string of currently executed test
        in the main VPP for logging purposes.
        """
        self._testMethodName = name
        self._testMethodDoc = doc
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Hawkmoth Test documentation build configuration file, created by
# sphinx-quickstart on Sun Nov 27 12:37:03 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../..'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['hawkmoth.cautodoc']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Hawkmoth Test'
copyright = u'2016, Jani Nikula'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'HawkmothTestdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'HawkmothTest.tex', u'Hawkmoth Test Documentation',
   u'Jani Nikula', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'hawkmothtest', u'Hawkmoth Test Documentation',
     [u'Jani Nikula'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'HawkmothTest', u'Hawkmoth Test Documentation',
   u'Jani Nikula', 'HawkmothTest', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Hawkmoth CAutoDoc
cautodoc_root = os.path.abspath('..')
<commit_msg>test: update the extension module name in the test sphinx project<commit_after># -*- coding: utf-8 -*-
#
# Hawkmoth Test documentation build configuration file, created by
# sphinx-quickstart on Sun Nov 27 12:37:03 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../..'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['hawkmoth']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Hawkmoth Test'
copyright = u'2016, Jani Nikula'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'HawkmothTestdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'HawkmothTest.tex', u'Hawkmoth Test Documentation',
   u'Jani Nikula', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'hawkmothtest', u'Hawkmoth Test Documentation',
     [u'Jani Nikula'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'HawkmothTest', u'Hawkmoth Test Documentation',
   u'Jani Nikula', 'HawkmothTest', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Hawkmoth CAutoDoc
cautodoc_root = os.path.abspath('..')
<|endoftext|>


<commit_before>"""Import timing information from icestorm generated SDF file and updates
timing in the architecture file.
"""

import argparse
from sdf_timing.sdfparse import parse as sdf_parse
import lxml.etree as ET
import re
import logging
from collections import namedtuple

class pin_map(namedtuple('pin_map', 'sdf arch is_clk')):
    """
    """

"""
arch_name  - (sdfcelltype, [pin_map ] )
"""

_arch_to_sdf = {
    'BLK_IG-LUT4': ('LogicCell40', [
        pin_map('in0', 'in[0]', False),
        pin_map('in1', 'in[1]', False),
        pin_map('in2', 'in[2]', False),
        pin_map('in3', 'in[3]', False),
        pin_map('lcout', 'out', False)
    ]
    ),
    'BLK_IG-SB_LUT4': ('LogicCell40', [
        pin_map('in0', 'I0', False),
        pin_map('in1', 'I1', False),
        pin_map('in2', 'I2', False),
        pin_map('in3', 'I3', False),
        pin_map('lcout', 'O', False)
    ]
    ),
    'SB_RAM256x16': ('SB_RAM40_4K', [
        pin_map('RDATA[0]', 'RDATA[0]', False),
        pin_map('RDATA[1]', 'RDATA[1]', False),
        pin_map('RDATA[2]', 'RDATA[2]', False),
        pin_map('RDATA[3]', 'RDATA[3]', False),
        pin_map('RDATA[4]', 'RDATA[4]', False),
        pin_map('RDATA[5]', 'RDATA[5]', False),
        pin_map('RDATA[6]', 'RDATA[6]', False),
        pin_map('RDATA[7]', 'RDATA[7]', False),
        pin_map('RDATA[8]', 'RDATA[8]', False),
        pin_map('RDATA[9]', 'RDATA[9]', False),
        pin_map('RDATA[10]', 'RDATA[10]', False),
        pin_map('RDATA[11]', 'RDATA[11]', False),
        pin_map('RDATA[12]', 'RDATA[12]', False),
        pin_map('RDATA[13]', 'RDATA[13]', False),
        pin_map('RDATA[14]', 'RDATA[14]', False),
        pin_map('RDATA[15]', 'RDATA[15]', False),
        pin_map('RCLK', 'RCLK', True),
        pin_map('RE', 'RE', False),
        pin_map('RCLKE', 'RCLKE', False),
        pin_map('RADDR[0]', 'RADDR[0]', False),
        pin_map('RADDR[1]', 'RADDR[1]', False),
        pin_map('RADDR[2]', 'RADDR[2]', False),
        pin_map('RADDR[3]', 'RADDR[3]', False),
        pin_map('RADDR[4]', 'RADDR[4]', False),
        pin_map('RADDR[5]', 'RADDR[5]', False),
        pin_map('RADDR[6]', 'RADDR[6]', False),
        pin_map('RADDR[7]', 'RADDR[7]', False),
        pin_map('RADDR[8]', 'RADDR[8]', False),
        pin_map('RADDR[9]', 'RADDR[9]', False),
        pin_map('RADDR[10]', 'RADDR[10]', False),

        pin_map('WDATA[0]', 'WDATA[0]', False),
        pin_map('WDATA[1]', 'WDATA[1]', False),
        pin_map('WDATA[2]', 'WDATA[2]', False),
        pin_map('WDATA[3]', 'WDATA[3]', False),
        pin_map('WDATA[4]', 'WDATA[4]', False),
        pin_map('WDATA[5]', 'WDATA[5]', False),
        pin_map('WDATA[6]', 'WDATA[6]', False),
        pin_map('WDATA[7]', 'WDATA[7]', False),
        pin_map('WDATA[8]', 'WDATA[8]', False),
        pin_map('WDATA[9]', 'WDATA[9]', False),
        pin_map('WDATA[10]', 'WDATA[10]', False),
        pin_map('WDATA[11]', 'WDATA[11]', False),
        pin_map('WDATA[12]', 'WDATA[12]', False),
        pin_map('WDATA[13]', 'WDATA[13]', False),
        pin_map('WDATA[14]', 'WDATA[14]', False),
        pin_map('WDATA[15]', 'WDATA[15]', False),
        pin_map('WCLK', 'WCLK', True),
        pin_map('WE', 'WE', False),
        pin_map('WCLKE', 'WCLKE', False),
        pin_map('WADDR[0]', 'WADDR[0]', False),
        pin_map('WADDR[1]', 'WADDR[1]', False),
        pin_map('WADDR[2]', 'WADDR[2]', False),
        pin_map('WADDR[3]', 'WADDR[3]', False),
        pin_map('WADDR[4]', 'WADDR[4]', False),
        pin_map('WADDR[5]', 'WADDR[5]', False),
        pin_map('WADDR[6]', 'WADDR[6]', False),
        pin_map('WADDR[7]', 'WADDR[7]', False),
        pin_map('WADDR[8]', 'WADDR[8]', False),
        pin_map('WADDR[9]', 'WADDR[9]', False),
        pin_map('WADDR[10]', 'WADDR[10]', False),

        pin_map('MASK[0]', 'MASK[0]', False),
        pin_map('MASK[1]', 'MASK[1]', False),
        pin_map('MASK[2]', 'MASK[2]', False),
        pin_map('MASK[3]', 'MASK[3]', False),
        pin_map('MASK[4]', 'MASK[4]', False),
        pin_map('MASK[5]', 'MASK[5]', False),
        pin_map('MASK[6]', 'MASK[6]', False),
        pin_map('MASK[7]', 'MASK[7]', False),
        pin_map('MASK[8]', 'MASK[8]', False),
        pin_map('MASK[9]', 'MASK[9]', False),
        pin_map('MASK[10]', 'MASK[10]', False),
        pin_map('MASK[11]', 'MASK[11]', False),
        pin_map('MASK[12]', 'MASK[12]', False),
        pin_map('MASK[13]', 'MASK[13]', False),
        pin_map('MASK[14]', 'MASK[14]', False),
        pin_map('MASK[15]', 'MASK[15]', False),

    ]
    ),
    'SB_CARRY': ('ICE_CARRY_IN_MUX', [
    ]
    )
}

def get_scale(timescale):
    """Convert sdf timescale to scale factor

    >>> get_scale('1.0 fs')
    1e-15

    >>> get_scale('1ps')
    1e-12

    >>> get_scale('10 ns')
    1e-08

    >>> get_scale('10.0 us')
    9.999999999999999e-06

    >>> get_scale('100.0ms')
    0.1

    >>> get_scale('100 s')
    100.0

    """
    mm = re.match(r'(10{0,2})(\.0)? *([munpf]?s)', timescale)
    sc_lut = {
        's': 1.0,
        'ms': 1e-3,
        'us': 1e-6,
        'ns': 1e-9,
        'ps': 1e-12,
        'fs': 1e-15,
    }
    ret = None
    if mm:
        base, _, sc = mm.groups()
        ret = int(base) * sc_lut[sc]
    return ret

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--read_sdf',
        type=argparse.FileType('r'),
        help='sdf file to read timing from'
    )
    parser.add_argument(
        '--read_arch_xml',
        type=argparse.FileType('r'),
        help='arch xml file to read and update/add timing'
    )
    parser.add_argument(
        '--write_arch_xml',
        type=argparse.FileType('w'),
        help='arch xml file to write with updaed timing calues'
    )

    parser.add_argument(
        '-v',
        type=bool,
        help='verbose output'
    )

    logging.basicConfig(level=logging.INFO)

    args = parser.parse_args()
    timing = sdf_parse(args.read_sdf.read())

    tree = ET.parse(args.read_arch_xml)

    scale = get_scale(timing['header']['timescale'])
    #logging.info('sdf scale set to', scale)

    # flatten cells to list of max
    flat_timing = dict()
    for cell_name, cell in timing['cells'].items():
        flat_timing[cell_name] = []
        for instance_key, instance in cell.items():
            assert instance_key == '*', \
               'For iCE40 expect only wildcard instance {} in cell {}'.format(
                   instance_key, cell_name
               )
            for _, path in instance.items():
                flat_timing[cell_name].append(path)

    for key, time_list in flat_timing.items():
        logging.debug(key)
        for delay in time_list:
            logging.debug(delay)

    # look up parsed sdf on in from_pin, to_pin, and type
    def lookup_timing(timing_list, type, to_pin, from_pin):
        ret = []
        for xx in timing_list:
            if type == xx['type'] and xx['to_pin'].startswith(
                    to_pin) and xx['from_pin'].startswith(from_pin):
                ret.append(xx)
            else:
                pass
        return ret

    def get_pessimistic(timing):
        vals = []
        for del_type in timing['delay_paths'].values():
            for dels in del_type.values():
                vals.append(dels)
        max_del = max(vals)
        return str(max_del * scale)

    # TODO: need to take max across negedge and posedge

    # remove all existing tags and warn on them

    # iterate over existing arch and update/add delay tags
    for el in tree.iter():
        if el.tag == 'pb_type':
            pb_name = el.attrib['name']
            # insert timing tags from SDF file
            if pb_name in _arch_to_sdf.keys():
                cell_name, pin_table = _arch_to_sdf.get(pb_name, None)
                for timing in flat_timing[cell_name]:
                    def try_translate_pin(table, timing, name):
                        res = [entry for entry in table if entry.sdf == timing[name]]
                        assert len(res) <= 1

                        if len(res) == 1:
                            return res[0]
                        else:
                            return None

                    if timing['type'] == 'hold' or timing['type'] == 'setup':
                        topin = try_translate_pin(pin_table, timing, 'to_pin')
                        frompin = try_translate_pin(pin_table, timing, 'from_pin')
                        if topin is None or frompin is None:
                            continue
                        attribs = {'clock': frompin.arch,
                                   'port': '{}.{}'.format(pb_name, topin.arch),
                                   'value': get_pessimistic(timing)
                        }
                        hold_setup_el = ET.SubElement(el, 'T_hold', attribs)
                        logging.info(ET.tostring(hold_setup_el))
                    elif timing['type'] == 'iopath':
                        topin = try_translate_pin(pin_table, timing, 'to_pin')
                        frompin = try_translate_pin(pin_table, timing, 'from_pin')
                        if topin is None or frompin is None:
                            continue
                        if frompin.is_clk:
                            attribs = {'clock': '{}'.format(frompin.arch),
                            'port': '{}.{}'.format(pb_name, topin.arch),
                            'max': get_pessimistic(timing)
                            }
                            iopath_el = ET.SubElement(el, 'T_clock_to_Q', attribs)
                        else:
                            attribs = {'in_port': '{}.{}'.format(pb_name, frompin.arch),
                            'out_port': '{}.{}'.format(pb_name, topin.arch),
                            'max': get_pessimistic(timing)
                            }
                            iopath_el = ET.SubElement(el, 'delay_constant', attribs)
                        logging.info(ET.tostring(iopath_el))
                    elif timing['type'] == 'recovery':
                        pass
                    elif timing['type'] == 'removal':
                        pass
                    # T_clock_to_Q

    tree.write(args.write_arch_xml.name)

if __name__ == '__main__':
    main()
<commit_msg>ice40: timing import update cell map<commit_after>"""Import timing information from icestorm generated SDF file and updates
timing in the architecture file.
"""

import argparse
from sdf_timing.sdfparse import parse as sdf_parse
import lxml.etree as ET
import re
import logging
from collections import namedtuple

class pin_map(namedtuple('pin_map', 'sdf arch is_clk')):
    """
    """

"""
arch_name  - (sdfcelltype, [pin_map ] )
"""

_arch_to_sdf = {
    'LUT4': ('LogicCell40', [
        pin_map('in0', 'in[0]', False),
        pin_map('in1', 'in[1]', False),
        pin_map('in2', 'in[2]', False),
        pin_map('in3', 'in[3]', False),
        pin_map('lcout', 'out', False)
    ]
    ),
    'SB_LUT4': ('LogicCell40', [
        pin_map('in0', 'I0', False),
        pin_map('in1', 'I1', False),
        pin_map('in2', 'I2', False),
        pin_map('in3', 'I3', False),
        pin_map('lcout', 'O', False)
    ]
    ),
    'SB_RAM256x16': ('SB_RAM40_4K', [
        pin_map('RDATA[0]', 'RDATA[0]', False),
        pin_map('RDATA[1]', 'RDATA[1]', False),
        pin_map('RDATA[2]', 'RDATA[2]', False),
        pin_map('RDATA[3]', 'RDATA[3]', False),
        pin_map('RDATA[4]', 'RDATA[4]', False),
        pin_map('RDATA[5]', 'RDATA[5]', False),
        pin_map('RDATA[6]', 'RDATA[6]', False),
        pin_map('RDATA[7]', 'RDATA[7]', False),
        pin_map('RDATA[8]', 'RDATA[8]', False),
        pin_map('RDATA[9]', 'RDATA[9]', False),
        pin_map('RDATA[10]', 'RDATA[10]', False),
        pin_map('RDATA[11]', 'RDATA[11]', False),
        pin_map('RDATA[12]', 'RDATA[12]', False),
        pin_map('RDATA[13]', 'RDATA[13]', False),
        pin_map('RDATA[14]', 'RDATA[14]', False),
        pin_map('RDATA[15]', 'RDATA[15]', False),
        pin_map('RCLK', 'RCLK', True),
        pin_map('RE', 'RE', False),
        pin_map('RCLKE', 'RCLKE', False),
        pin_map('RADDR[0]', 'RADDR[0]', False),
        pin_map('RADDR[1]', 'RADDR[1]', False),
        pin_map('RADDR[2]', 'RADDR[2]', False),
        pin_map('RADDR[3]', 'RADDR[3]', False),
        pin_map('RADDR[4]', 'RADDR[4]', False),
        pin_map('RADDR[5]', 'RADDR[5]', False),
        pin_map('RADDR[6]', 'RADDR[6]', False),
        pin_map('RADDR[7]', 'RADDR[7]', False),
        pin_map('RADDR[8]', 'RADDR[8]', False),
        pin_map('RADDR[9]', 'RADDR[9]', False),
        pin_map('RADDR[10]', 'RADDR[10]', False),

        pin_map('WDATA[0]', 'WDATA[0]', False),
        pin_map('WDATA[1]', 'WDATA[1]', False),
        pin_map('WDATA[2]', 'WDATA[2]', False),
        pin_map('WDATA[3]', 'WDATA[3]', False),
        pin_map('WDATA[4]', 'WDATA[4]', False),
        pin_map('WDATA[5]', 'WDATA[5]', False),
        pin_map('WDATA[6]', 'WDATA[6]', False),
        pin_map('WDATA[7]', 'WDATA[7]', False),
        pin_map('WDATA[8]', 'WDATA[8]', False),
        pin_map('WDATA[9]', 'WDATA[9]', False),
        pin_map('WDATA[10]', 'WDATA[10]', False),
        pin_map('WDATA[11]', 'WDATA[11]', False),
        pin_map('WDATA[12]', 'WDATA[12]', False),
        pin_map('WDATA[13]', 'WDATA[13]', False),
        pin_map('WDATA[14]', 'WDATA[14]', False),
        pin_map('WDATA[15]', 'WDATA[15]', False),
        pin_map('WCLK', 'WCLK', True),
        pin_map('WE', 'WE', False),
        pin_map('WCLKE', 'WCLKE', False),
        pin_map('WADDR[0]', 'WADDR[0]', False),
        pin_map('WADDR[1]', 'WADDR[1]', False),
        pin_map('WADDR[2]', 'WADDR[2]', False),
        pin_map('WADDR[3]', 'WADDR[3]', False),
        pin_map('WADDR[4]', 'WADDR[4]', False),
        pin_map('WADDR[5]', 'WADDR[5]', False),
        pin_map('WADDR[6]', 'WADDR[6]', False),
        pin_map('WADDR[7]', 'WADDR[7]', False),
        pin_map('WADDR[8]', 'WADDR[8]', False),
        pin_map('WADDR[9]', 'WADDR[9]', False),
        pin_map('WADDR[10]', 'WADDR[10]', False),

        pin_map('MASK[0]', 'MASK[0]', False),
        pin_map('MASK[1]', 'MASK[1]', False),
        pin_map('MASK[2]', 'MASK[2]', False),
        pin_map('MASK[3]', 'MASK[3]', False),
        pin_map('MASK[4]', 'MASK[4]', False),
        pin_map('MASK[5]', 'MASK[5]', False),
        pin_map('MASK[6]', 'MASK[6]', False),
        pin_map('MASK[7]', 'MASK[7]', False),
        pin_map('MASK[8]', 'MASK[8]', False),
        pin_map('MASK[9]', 'MASK[9]', False),
        pin_map('MASK[10]', 'MASK[10]', False),
        pin_map('MASK[11]', 'MASK[11]', False),
        pin_map('MASK[12]', 'MASK[12]', False),
        pin_map('MASK[13]', 'MASK[13]', False),
        pin_map('MASK[14]', 'MASK[14]', False),
        pin_map('MASK[15]', 'MASK[15]', False),

    ]
    ),
    'SB_CARRY': ('ICE_CARRY_IN_MUX', [
    ]
    )
}

def get_scale(timescale):
    """Convert sdf timescale to scale factor

    >>> get_scale('1.0 fs')
    1e-15

    >>> get_scale('1ps')
    1e-12

    >>> get_scale('10 ns')
    1e-08

    >>> get_scale('10.0 us')
    9.999999999999999e-06

    >>> get_scale('100.0ms')
    0.1

    >>> get_scale('100 s')
    100.0

    """
    mm = re.match(r'(10{0,2})(\.0)? *([munpf]?s)', timescale)
    sc_lut = {
        's': 1.0,
        'ms': 1e-3,
        'us': 1e-6,
        'ns': 1e-9,
        'ps': 1e-12,
        'fs': 1e-15,
    }
    ret = None
    if mm:
        base, _, sc = mm.groups()
        ret = int(base) * sc_lut[sc]
    return ret

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--read_sdf',
        type=argparse.FileType('r'),
        help='sdf file to read timing from'
    )
    parser.add_argument(
        '--read_arch_xml',
        type=argparse.FileType('r'),
        help='arch xml file to read and update/add timing'
    )
    parser.add_argument(
        '--write_arch_xml',
        type=argparse.FileType('w'),
        help='arch xml file to write with updaed timing calues'
    )

    parser.add_argument(
        '-v',
        type=bool,
        help='verbose output'
    )

    logging.basicConfig(level=logging.INFO)

    args = parser.parse_args()
    timing = sdf_parse(args.read_sdf.read())

    tree = ET.parse(args.read_arch_xml, ET.XMLParser(remove_blank_text=True))

    scale = get_scale(timing['header']['timescale'])
    #logging.info('sdf scale set to', scale)

    # flatten cells to list of max
    flat_timing = dict()
    for cell_name, cell in timing['cells'].items():
        flat_timing[cell_name] = []
        for instance_key, instance in cell.items():
            assert instance_key == '*', \
               'For iCE40 expect only wildcard instance {} in cell {}'.format(
                   instance_key, cell_name
               )
            for _, path in instance.items():
                flat_timing[cell_name].append(path)

    for key, time_list in flat_timing.items():
        logging.debug(key)
        for delay in time_list:
            logging.debug(delay)

    # look up parsed sdf on in from_pin, to_pin, and type
    def lookup_timing(timing_list, type, to_pin, from_pin):
        ret = []
        for xx in timing_list:
            if type == xx['type'] and xx['to_pin'].startswith(
                    to_pin) and xx['from_pin'].startswith(from_pin):
                ret.append(xx)
            else:
                pass
        return ret

    def get_pessimistic(timing):
        vals = []
        for del_type in timing['delay_paths'].values():
            for dels in del_type.values():
                vals.append(dels)
        max_del = max(vals)
        return str(max_del * scale)

    # TODO: need to take max across negedge and posedge

    # remove all existing tags and warn on them

    # iterate over existing arch and update/add delay tags
    #root = tree.getroot()
    for el in tree.iter():
        if el.tag == 'pb_type':
            pb_name = el.attrib['name']
            # insert timing tags from SDF file
            if pb_name in _arch_to_sdf.keys():
                cell_name, pin_table = _arch_to_sdf.get(pb_name, None)
                for timing in flat_timing[cell_name]:
                    def try_translate_pin(table, timing, name):
                        res = [entry for entry in table if entry.sdf == timing[name]]
                        assert len(res) <= 1

                        if len(res) == 1:
                            return res[0]
                        else:
                            return None

                    if timing['type'] == 'hold' or timing['type'] == 'setup':
                        topin = try_translate_pin(pin_table, timing, 'to_pin')
                        frompin = try_translate_pin(pin_table, timing, 'from_pin')
                        if topin is None or frompin is None:
                            continue
                        attribs = {'clock': frompin.arch,
                                   'port': '{}.{}'.format(pb_name, topin.arch),
                                   'value': get_pessimistic(timing)
                        }
                        hold_setup_el = ET.SubElement(el, 'T_{}'.format(timing['type']), attribs)
                        # hold_setup_el.tail = '\n'
                        logging.info(ET.tostring(hold_setup_el))
                    elif timing['type'] == 'iopath':
                        topin = try_translate_pin(pin_table, timing, 'to_pin')
                        frompin = try_translate_pin(pin_table, timing, 'from_pin')
                        if topin is None or frompin is None:
                            continue
                        if frompin.is_clk:
                            attribs = {'clock': '{}'.format(frompin.arch),
                            'port': '{}.{}'.format(pb_name, topin.arch),
                            'max': get_pessimistic(timing)
                            }
                            iopath_el = ET.SubElement(el, 'T_clock_to_Q', attribs)
                            # iopath_el.tail = '\n'
                        else:
                            attribs = {'in_port': '{}.{}'.format(pb_name, frompin.arch),
                            'out_port': '{}.{}'.format(pb_name, topin.arch),
                            'max': get_pessimistic(timing)
                            }
                            iopath_el = ET.SubElement(el, 'delay_constant', attribs)
                            # iopath_el.tail = '\n'
                        logging.info(ET.tostring(iopath_el))
                    elif timing['type'] == 'recovery':
                        pass
                    elif timing['type'] == 'removal':
                        pass

    xml_str = ET.tostring(tree, pretty_print=True).decode('utf-8')
    args.write_arch_xml.write(xml_str)

if __name__ == '__main__':
    main()
<|endoftext|>