<commit_before>#!/usr/bin/env python

"""
Based on http://groups.google.com/group/dropio-api/web/full-api-documentation
"""

__author__ = 'jimmyorr@gmail.com (Jimmy Orr)'

import os.path
import re
from StringIO import StringIO
import urllib
import urllib2

import pycurl
try: import json
except ImportError: import simplejson as json

from resource import Asset, Drop, Link, Note

API_VERSION = '1.0'
API_FORMAT = 'json'

API_BASE_URL = 'http://api.drop.io/'
FILE_UPLOAD_URL = 'http://assets.drop.io/upload'

DROPS = 'drops/'
ASSETS = '/assets/'
COMMENTS = '/comments/'
SEND_TO = '/send_to'

DROP_NAME_PATTERN = '^[a-zA-Z0-9_]+$'

class DropIoClient(object):
    """Client for the Drop.io service."""

    def __init__(self, api_key):
        self.__base_params_dict = {}
        self.__base_params_dict['api_key'] = api_key
        self.__base_params_dict['version'] = API_VERSION
        self.__base_params_dict['format'] = API_FORMAT

    def __get(self, base_url, params_dict):
        params = urllib.urlencode(params_dict)
        f = urllib2.urlopen(base_url + '?' + params)
        body_dict = json.load(f)
        f.close()
        return body_dict

    def __post(self, url, params_dict):
        params = urllib.urlencode(params_dict)
        f = urllib2.urlopen(url, params)
        body_dict = json.load(f)
        f.close()
        return body_dict

    def __put(self, url, params_dict):
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url, data=json.dumps(params_dict))
        request.add_header('Content-Type', 'application/json')
        request.get_method = lambda: 'PUT'
        f = opener.open(request)
        body_dict = json.load(f)
        f.close()
        opener.close()
        return body_dict

    def __delete(self, url, params_dict):
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url, data=json.dumps(params_dict))
        request.add_header('Content-Type', 'application/json')
        request.get_method = lambda: 'DELETE'
        f = opener.open(request)
        body_dict = json.load(f)
        f.close()
        opener.close()
        return body_dict

    def __curl_post(self, url, params_dict):
        # TODO: wish there was a better way to do this...preferably without curl
        def __parse_headers(headers_str):
            cookies_dict = {}
            headers_dict = {}

            lines = re.split('\r?\n', headers_str)
            for line in lines:
                try:
                    name, value = line.split(': ', 1)
                    if 'Set-Cookie' == name:
                        match = re.search('^([^=]+)=([^;]+)*', value)
                        if match:
                            cookies_dict[match.group(1)] = match.group(2)
                    else:
                        headers_dict[name] = value
                except ValueError:
                    pass

            return headers_dict

        c = pycurl.Curl()
        headers = StringIO()
        body = StringIO()

        c.setopt(pycurl.HEADERFUNCTION, headers.write)
        c.setopt(pycurl.WRITEFUNCTION, body.write)
        c.setopt(pycurl.URL, str(url))
        c.setopt(pycurl.HTTPPOST, params_dict.items())

        c.perform()
        c.close()

        headers.seek(0)
        headers_dict = __parse_headers(headers.read())
        headers.close()

        body.seek(0)
        body_dict = json.load(body)
        body.close()

        if headers_dict.get('Status') != '200 OK':
            response_dict = body_dict.get('response')
            if response_dict and getattr(response_dict, 'get'):
                raise RuntimeError(response_dict.get('message'))

        return body_dict

    def __check_drop_name(self, drop_name):
        # TODO: throw more specific exception
        assert re.search(DROP_NAME_PATTERN, drop_name) is not None

    def __asset_dict_to_asset(self, asset_dict):
        # TODO: this isn't ideal...
        asset = None
        if asset_dict.has_key('contents'):
            asset = Note(asset_dict)
        elif asset_dict.has_key('url'):
            asset = Link(asset_dict)
        else:
            asset = Asset(asset_dict)
        return asset

    ################
    # DROP RESOURCE
    ################

    def create_drop(self, drop_name=None):
        """
        Returns:
            dropio.resource.Drop
        """
        if drop_name is not None:
            self.__check_drop_name(drop_name)

        params_dict = {}
        if drop_name is not None:
            params_dict['name'] = drop_name
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS
        drop_dict = self.__post(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def get_drop(self, drop_name, token=None):
        """
        Returns:
            dropio.resource.Drop
        """
        assert drop_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name
        drop_dict = self.__get(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def update_drop(self, drop, token):
        """
        Returns:
            dropio.resource.Drop
        """
        assert drop is not None
        assert token is not None

        params_dict = {}
        params_dict['token'] = token
        if drop.guests_can_comment is not None:
            params_dict['guests_can_comment'] = drop.guests_can_comment
        if drop.guests_can_add is not None:
            params_dict['guests_can_add'] = drop.guests_can_add
        if drop.guests_can_delete is not None:
            params_dict['guests_can_delete'] = drop.guests_can_delete
        if drop.expiration_length is not None:
            params_dict['expiration_length'] = drop.expiration_length
        if drop.password is not None:
            params_dict['password'] = drop.password
        if drop.admin_password is not None:
            params_dict['admin_password'] = drop.admin_password
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop.name
        drop_dict = self.__put(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def delete_drop(self, drop_name, token):
        assert drop_name is not None
        assert token is not None

        params_dict = {}
        params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name
        self.__delete(url, params_dict)

        return

    #################
    # ASSET RESOURCE
    #################

    def create_link(self, drop_name, link_url,
                    title=None, description=None, token=None):
        """
        Returns:
            dropio.resource.Link
        """
        assert drop_name is not None
        assert link_url is not None

        params_dict = {}
        params_dict['url'] = link_url
        if title is not None:
            params_dict['title'] = title
        if description is not None:
            params_dict['description'] = description
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS
        link_dict = self.__post(url, params_dict)
        link = Link(link_dict)

        return link

    def create_note(self, drop_name, contents, title=None, token=None):
        """
        Returns:
            dropio.resource.Note
        """
        assert drop_name is not None
        assert contents is not None

        params_dict = {}
        params_dict['contents'] = contents
        if title is not None:
            params_dict['title'] = title
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS
        note_dict = self.__post(url, params_dict)
        note = Note(note_dict)

        return note

    def create_file(self, drop_name, file_name, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert file_name is not None
        assert os.path.isfile(file_name) is True

        params_dict = {}
        params_dict['drop_name'] = drop_name
        params_dict['file'] = (pycurl.FORM_FILE, file_name)
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = FILE_UPLOAD_URL
        asset_dict = self.__curl_post(url, params_dict)
        asset = Asset(asset_dict)

        return asset

    def get_asset_list(self, drop_name, token=None):
        """
        Returns:
            list of dropio.resource.Asset
        """
        assert drop_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        # TODO: paginate through asset list for > 30 assets
        url = API_BASE_URL + DROPS + drop_name + ASSETS
        asset_dicts = self.__get(url, params_dict)

        for asset_dict in asset_dicts:
            yield Asset(asset_dict)

    def get_asset(self, drop_name, asset_name, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert asset_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name
        asset_dict = self.__get(url, params_dict)
        asset = self.__asset_dict_to_asset(asset_dict)

        return asset

    def update_asset(self, drop_name, asset, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert asset is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        if asset.title is not None:
            params_dict['title'] = asset.title
        if asset.description is not None:
            params_dict['description'] = asset.description
        if hasattr(asset, 'url') and asset.url is not None:
            params_dict['url'] = asset.url
        if hasattr(asset, 'contents') and asset.contents is not None:
            params_dict['contents'] = asset.contents
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset.name
        asset_dict = self.__put(url, params_dict)
        asset = self.__asset_dict_to_asset(asset_dict)

        return asset

    def delete_asset(self, drop_name, asset_name, token=None):
        assert drop_name is not None
        assert asset_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name
        self.__delete(url, params_dict)

        return

    def __send_asset(self, drop_name, asset_name, medium, params_dict, token=None):
        assert drop_name is not None
        assert asset_name is not None

        params_dict['medium'] = medium
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name + SEND_TO
        self.__post(url, params_dict)

        return

    def send_asset_to_fax(self, drop_name, asset_name, fax_number, token=None):
        assert fax_number is not None

        params_dict = {}
        params_dict['fax_number'] = fax_number
        self.__send_asset(drop_name, asset_name, 'fax', params_dict, token)

        return

    def send_asset_to_drop(self, drop_name, asset_name, drop_name_dest, token=None):
        assert drop_name_dest is not None

        params_dict = {}
        params_dict['drop_name'] = drop_name_dest
        self.__send_asset(drop_name, asset_name, 'drop', params_dict, token)

        return

    def send_asset_to_email(self, drop_name, asset_name, emails, message=None, token=None):
        assert emails is not None

        params_dict = {}
        params_dict['emails'] = emails
        if message is not None:
            params_dict['message'] = message
        self.__send_asset(drop_name, asset_name, 'email', params_dict, token)

        return

    ###################
    # COMMENT RESOURCE
    ###################

    def get_comment_list(self, drop_name, asset_name, token=None):
        """
        Returns:
            list of dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def create_comment(self, drop_name, asset_name, contents, token=None):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def get_comment(self, drop_name, asset_name, comment_id, token=None):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def update_comment(self, drop_name, asset_name, comment, token):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def delete_comment(self, drop_name, asset_name, comment_id, token):
        """
        Returns:
            ???
        """
        # TODO: implement me
        raise NotImplementedError()
<commit_msg>Let the webservice determine what is a valid drop name, rather than duplicate checks on client<commit_after>#!/usr/bin/env python

"""
Based on http://groups.google.com/group/dropio-api/web/full-api-documentation
"""

__author__ = 'jimmyorr@gmail.com (Jimmy Orr)'

import os.path
import re
from StringIO import StringIO
import urllib
import urllib2

import pycurl
try: import json
except ImportError: import simplejson as json

from resource import Asset, Drop, Link, Note

API_VERSION = '1.0'
API_FORMAT = 'json'

API_BASE_URL = 'http://api.drop.io/'
FILE_UPLOAD_URL = 'http://assets.drop.io/upload'

DROPS = 'drops/'
ASSETS = '/assets/'
COMMENTS = '/comments/'
SEND_TO = '/send_to'

class DropIoClient(object):
    """Client for the Drop.io service."""

    def __init__(self, api_key):
        self.__base_params_dict = {}
        self.__base_params_dict['api_key'] = api_key
        self.__base_params_dict['version'] = API_VERSION
        self.__base_params_dict['format'] = API_FORMAT

    def __get(self, base_url, params_dict):
        params = urllib.urlencode(params_dict)
        f = urllib2.urlopen(base_url + '?' + params)
        body_dict = json.load(f)
        f.close()
        return body_dict

    def __post(self, url, params_dict):
        params = urllib.urlencode(params_dict)
        f = urllib2.urlopen(url, params)
        body_dict = json.load(f)
        f.close()
        return body_dict

    def __put(self, url, params_dict):
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url, data=json.dumps(params_dict))
        request.add_header('Content-Type', 'application/json')
        request.get_method = lambda: 'PUT'
        f = opener.open(request)
        body_dict = json.load(f)
        f.close()
        opener.close()
        return body_dict

    def __delete(self, url, params_dict):
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url, data=json.dumps(params_dict))
        request.add_header('Content-Type', 'application/json')
        request.get_method = lambda: 'DELETE'
        f = opener.open(request)
        body_dict = json.load(f)
        f.close()
        opener.close()
        return body_dict

    def __curl_post(self, url, params_dict):
        # TODO: wish there was a better way to do this...preferably without curl
        def __parse_headers(headers_str):
            cookies_dict = {}
            headers_dict = {}

            lines = re.split('\r?\n', headers_str)
            for line in lines:
                try:
                    name, value = line.split(': ', 1)
                    if 'Set-Cookie' == name:
                        match = re.search('^([^=]+)=([^;]+)*', value)
                        if match:
                            cookies_dict[match.group(1)] = match.group(2)
                    else:
                        headers_dict[name] = value
                except ValueError:
                    pass

            return headers_dict

        c = pycurl.Curl()
        headers = StringIO()
        body = StringIO()

        c.setopt(pycurl.HEADERFUNCTION, headers.write)
        c.setopt(pycurl.WRITEFUNCTION, body.write)
        c.setopt(pycurl.URL, str(url))
        c.setopt(pycurl.HTTPPOST, params_dict.items())

        c.perform()
        c.close()

        headers.seek(0)
        headers_dict = __parse_headers(headers.read())
        headers.close()

        body.seek(0)
        body_dict = json.load(body)
        body.close()

        if headers_dict.get('Status') != '200 OK':
            response_dict = body_dict.get('response')
            if response_dict and getattr(response_dict, 'get'):
                raise RuntimeError(response_dict.get('message'))

        return body_dict

    def __asset_dict_to_asset(self, asset_dict):
        # TODO: this isn't ideal...
        asset = None
        if asset_dict.has_key('contents'):
            asset = Note(asset_dict)
        elif asset_dict.has_key('url'):
            asset = Link(asset_dict)
        else:
            asset = Asset(asset_dict)
        return asset

    ################
    # DROP RESOURCE
    ################

    def create_drop(self, drop_name=None):
        """
        Returns:
            dropio.resource.Drop
        """

        params_dict = {}
        if drop_name is not None:
            params_dict['name'] = drop_name
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS
        drop_dict = self.__post(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def get_drop(self, drop_name, token=None):
        """
        Returns:
            dropio.resource.Drop
        """
        assert drop_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name
        drop_dict = self.__get(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def update_drop(self, drop, token):
        """
        Returns:
            dropio.resource.Drop
        """
        assert drop is not None
        assert token is not None

        params_dict = {}
        params_dict['token'] = token
        if drop.guests_can_comment is not None:
            params_dict['guests_can_comment'] = drop.guests_can_comment
        if drop.guests_can_add is not None:
            params_dict['guests_can_add'] = drop.guests_can_add
        if drop.guests_can_delete is not None:
            params_dict['guests_can_delete'] = drop.guests_can_delete
        if drop.expiration_length is not None:
            params_dict['expiration_length'] = drop.expiration_length
        if drop.password is not None:
            params_dict['password'] = drop.password
        if drop.admin_password is not None:
            params_dict['admin_password'] = drop.admin_password
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop.name
        drop_dict = self.__put(url, params_dict)
        drop = Drop(drop_dict)

        return drop

    def delete_drop(self, drop_name, token):
        assert drop_name is not None
        assert token is not None

        params_dict = {}
        params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name
        self.__delete(url, params_dict)

        return

    #################
    # ASSET RESOURCE
    #################

    def create_link(self, drop_name, link_url,
                    title=None, description=None, token=None):
        """
        Returns:
            dropio.resource.Link
        """
        assert drop_name is not None
        assert link_url is not None

        params_dict = {}
        params_dict['url'] = link_url
        if title is not None:
            params_dict['title'] = title
        if description is not None:
            params_dict['description'] = description
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS
        link_dict = self.__post(url, params_dict)
        link = Link(link_dict)

        return link

    def create_note(self, drop_name, contents, title=None, token=None):
        """
        Returns:
            dropio.resource.Note
        """
        assert drop_name is not None
        assert contents is not None

        params_dict = {}
        params_dict['contents'] = contents
        if title is not None:
            params_dict['title'] = title
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS
        note_dict = self.__post(url, params_dict)
        note = Note(note_dict)

        return note

    def create_file(self, drop_name, file_name, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert file_name is not None
        assert os.path.isfile(file_name) is True

        params_dict = {}
        params_dict['drop_name'] = drop_name
        params_dict['file'] = (pycurl.FORM_FILE, file_name)
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = FILE_UPLOAD_URL
        asset_dict = self.__curl_post(url, params_dict)
        asset = Asset(asset_dict)

        return asset

    def get_asset_list(self, drop_name, token=None):
        """
        Returns:
            list of dropio.resource.Asset
        """
        assert drop_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        # TODO: paginate through asset list for > 30 assets
        url = API_BASE_URL + DROPS + drop_name + ASSETS
        asset_dicts = self.__get(url, params_dict)

        for asset_dict in asset_dicts:
            yield Asset(asset_dict)

    def get_asset(self, drop_name, asset_name, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert asset_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name
        asset_dict = self.__get(url, params_dict)
        asset = self.__asset_dict_to_asset(asset_dict)

        return asset

    def update_asset(self, drop_name, asset, token=None):
        """
        Returns:
            dropio.resource.Asset
        """
        assert drop_name is not None
        assert asset is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        if asset.title is not None:
            params_dict['title'] = asset.title
        if asset.description is not None:
            params_dict['description'] = asset.description
        if hasattr(asset, 'url') and asset.url is not None:
            params_dict['url'] = asset.url
        if hasattr(asset, 'contents') and asset.contents is not None:
            params_dict['contents'] = asset.contents
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset.name
        asset_dict = self.__put(url, params_dict)
        asset = self.__asset_dict_to_asset(asset_dict)

        return asset

    def delete_asset(self, drop_name, asset_name, token=None):
        assert drop_name is not None
        assert asset_name is not None

        params_dict = {}
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name
        self.__delete(url, params_dict)

        return

    def __send_asset(self, drop_name, asset_name, medium, params_dict, token=None):
        assert drop_name is not None
        assert asset_name is not None

        params_dict['medium'] = medium
        if token is not None:
            params_dict['token'] = token
        params_dict.update(self.__base_params_dict)

        url = API_BASE_URL + DROPS + drop_name + ASSETS + asset_name + SEND_TO
        self.__post(url, params_dict)

        return

    def send_asset_to_fax(self, drop_name, asset_name, fax_number, token=None):
        assert fax_number is not None

        params_dict = {}
        params_dict['fax_number'] = fax_number
        self.__send_asset(drop_name, asset_name, 'fax', params_dict, token)

        return

    def send_asset_to_drop(self, drop_name, asset_name, drop_name_dest, token=None):
        assert drop_name_dest is not None

        params_dict = {}
        params_dict['drop_name'] = drop_name_dest
        self.__send_asset(drop_name, asset_name, 'drop', params_dict, token)

        return

    def send_asset_to_email(self, drop_name, asset_name, emails, message=None, token=None):
        assert emails is not None

        params_dict = {}
        params_dict['emails'] = emails
        if message is not None:
            params_dict['message'] = message
        self.__send_asset(drop_name, asset_name, 'email', params_dict, token)

        return

    ###################
    # COMMENT RESOURCE
    ###################

    def get_comment_list(self, drop_name, asset_name, token=None):
        """
        Returns:
            list of dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def create_comment(self, drop_name, asset_name, contents, token=None):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def get_comment(self, drop_name, asset_name, comment_id, token=None):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def update_comment(self, drop_name, asset_name, comment, token):
        """
        Returns:
            dropio.resource.Comment
        """
        # TODO: implement me
        raise NotImplementedError()

    def delete_comment(self, drop_name, asset_name, comment_id, token):
        """
        Returns:
            ???
        """
        # TODO: implement me
        raise NotImplementedError()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''
eval.slashtags_eval -- pos tag evaluator

eval.slashtags_eval is a script to evaluate pos-tagged files.

It defines a main method that scores tag accuracy.

@author:     Ryan Georgi

@copyright:  2013 Ryan Georgi. All rights reserved.

@license:    MIT License

@contact:    rgeorgi@uw.edu
@deffield    updated: Updated
'''

import sys
import os

from argparse import ArgumentParser
from utils import ConfigFile
import re
from eval.EvalException import POSEvalException, EvalException
from utils.TwoLevelCountDict import TwoLevelCountDict, POSEvalDict
from corpora.POSCorpus import POSCorpus

__all__ = []
__version__ = 0.1
__date__ = '2013-08-26'
__updated__ = '2013-08-26'

#===============================================================================
#
#===============================================================================

def slashtags_eval(goldpath, testpath, delimeter, out_f=sys.stdout, tagmap=None):

	gold_c = POSCorpus.read_slashtags(goldpath)
	test_c = POSCorpus.read_slashtags(testpath)

	poseval(test_c, gold_c, out_f)

#===============================================================================
#
#===============================================================================

def simple_tagger_eval(eval_path, gold_path, out_f = sys.stdout, csv=True):

	eval_c = POSCorpus.read_simpletagger(eval_path)
	gold_c = POSCorpus.read_simpletagger(gold_path)

	poseval(eval_c, gold_c, out_f)

def poseval(eval_sents, gold_sents, out_f = sys.stdout, csv=True, ansi=False, greedy_1_to_1=True, greedy_n_to_1=True):
	if len(eval_sents) != len(gold_sents):
		raise EvalException('Number of eval sents does not match number of gold sents.')

	#===========================================================================
	# Set up counters
	#===========================================================================
	c = POSEvalDict()
	d = POSEvalDict()

	i = 1

	for eval_sent, gold_sent in zip(eval_sents, gold_sents):

		# Check whether the whole sentence is correct.
		sent_correct = True

		if len(eval_sent) != len(gold_sent):
			raise EvalException('Number of tokens for sent #%d is unequal' % i)

		for eval_token, gold_token in zip(eval_sent, gold_sent):

			gold_label = gold_token.label
			eval_label = eval_token.label

			# Kludgy way to make sure all the assigned
			# labels end up getting seen.
			c[eval_label].add(eval_label, 0)

			c[gold_label].add(eval_label, 1)
			d[eval_label].add(gold_label, 1)

			# If one of the labels does not match,
			# the sentence does not match.
			if gold_label != eval_label:
				sent_correct = False

		# If the sentence matches it, count it...

	#===========================================================================
	# Now, evaluate based on the gold-to-eval labels
	#===========================================================================
	eval_print_helper(out_f, 'STANDARD', c, ansi, csv)

	if greedy_1_to_1:
		c.greedy_1_to_1()
		eval_print_helper(out_f, 'GREEDY 1-to-1', c, ansi, csv)

	if greedy_n_to_1:
		c.greedy_n_to_1()
		eval_print_helper(out_f, 'GREEDY N-to-1', c, ansi, csv)

	return c.accuracy()

def eval_print_helper(out_f, title, c, ansi, csv):
	out_f.write('='*80+'\n')
	out_f.write('%s:\n' % title + '-'*80+'\n' + c.error_matrix(ansi=ansi, csv=csv))
	out_f.write('-'*80+'\nOVERALL ACCURACY: %.2f\n' % c.accuracy() + '='*80+'\n')

if __name__ == '__main__':

	# setup option parser
	parser = ArgumentParser()
	parser.add_argument("-c", "--conf", dest="conf", help="set input path [default: %default]", metavar="FILE")
	parser.add_argument('-g', '--gold', dest='gold', help='Specify gold file without conf file.')
	parser.add_argument('-t', '--test', dest='test', help='Specify test file without conf file.')
	parser.add_argument('-d', '--delimeter', dest='delimeter', help='Specify the delimeter', default='/')

	# set defaults
	args = parser.parse_args()

	if not args.conf:
		if not args.gold and args.test:
			sys.stderr.write('Either the conf file or gold and opts file must be specified.')
			parser.print_help()
			sys.exit()
		else:
			gold = args.gold
			test = args.test
			delimeter = args.delimeter
	else:
		c = ConfigFile.ConfigFile(args.conf)
		gold = c['goldfile']
		test = c['testfile']
		delimeter = c.get('delimeter', '/')

		# Default to no tagmap, unless
		# specified in the config file.
		tagmap = c.get('tagmap')

	if gold and test and delimeter:
		slashtags_eval(gold, test, delimeter, tagmap)
	else:
		sys.stderr.write('Arguments missing.')
		sys.exit()
		<commit_msg>Adding some code to turn printing the confusion matrix off in poseval.<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''
eval.slashtags_eval -- pos tag evaluator

eval.slashtags_eval is a script to evaluate pos-tagged files.

It defines a main method that scores tag accuracy.

@author:     Ryan Georgi

@copyright:  2013 Ryan Georgi. All rights reserved.

@license:    MIT License

@contact:    rgeorgi@uw.edu
@deffield    updated: Updated
'''

import sys
import os

from argparse import ArgumentParser
from utils import ConfigFile
import re
from eval.EvalException import POSEvalException, EvalException
from utils.TwoLevelCountDict import TwoLevelCountDict, POSEvalDict
from corpora.POSCorpus import POSCorpus

__all__ = []
__version__ = 0.1
__date__ = '2013-08-26'
__updated__ = '2013-08-26'

#===============================================================================
#
#===============================================================================

def slashtags_eval(goldpath, testpath, delimeter, out_f=sys.stdout, tagmap=None):

	gold_c = POSCorpus.read_slashtags(goldpath)
	test_c = POSCorpus.read_slashtags(testpath)

	poseval(test_c, gold_c, out_f)

#===============================================================================
#
#===============================================================================

def simple_tagger_eval(eval_path, gold_path, out_f = sys.stdout, csv=True):

	eval_c = POSCorpus.read_simpletagger(eval_path)
	gold_c = POSCorpus.read_simpletagger(gold_path)

	poseval(eval_c, gold_c, out_f)

def poseval(eval_sents, gold_sents, out_f = sys.stdout, csv=True, ansi=False, greedy_1_to_1=False, greedy_n_to_1=False, matrix=False):
	if len(eval_sents) != len(gold_sents):
		raise EvalException('Number of eval sents does not match number of gold sents.')

	#===========================================================================
	# Set up counters
	#===========================================================================
	c = POSEvalDict()
	d = POSEvalDict()

	i = 1

	for eval_sent, gold_sent in zip(eval_sents, gold_sents):

		# Check whether the whole sentence is correct.
		sent_correct = True

		if len(eval_sent) != len(gold_sent):
			raise EvalException('Number of tokens for sent #%d is unequal' % i)

		for eval_token, gold_token in zip(eval_sent, gold_sent):

			gold_label = str(gold_token.label)
			eval_label = str(eval_token.label)

			# Kludgy way to make sure all the assigned
			# labels end up getting seen.
			c[eval_label].add(eval_label, 0)

			c[gold_label].add(eval_label, 1)
			d[eval_label].add(gold_label, 1)

			# If one of the labels does not match,
			# the sentence does not match.
			if gold_label != eval_label:
				sent_correct = False

		# If the sentence matches it, count it...

	#===========================================================================
	# Now, evaluate based on the gold-to-eval labels
	#===========================================================================
	eval_print_helper(out_f, 'STANDARD', matrix, c, ansi, csv)

	if greedy_1_to_1:
		c.greedy_1_to_1()
		eval_print_helper(out_f, 'GREEDY 1-to-1',  matrix, c, ansi, csv)

	if greedy_n_to_1:
		c.greedy_n_to_1()
		eval_print_helper(out_f, 'GREEDY N-to-1', matrix, c, ansi, csv)

	return c

def eval_print_helper(out_f, title, matrix, c, ansi, csv):
	out_f.write('='*80+'\n')
	out_f.write('%s:\n' % title + '-'*80+'\n')
	if matrix:
		out_f.write(c.error_matrix(ansi=ansi, csv=csv))
		out_f.write('-'*80+'\n')
	out_f.write('OVERALL ACCURACY: %.2f\n' % c.accuracy() + '='*80+'\n')

if __name__ == '__main__':

	# setup option parser
	parser = ArgumentParser()
	parser.add_argument("-c", "--conf", dest="conf", help="set input path [default: %default]", metavar="FILE")
	parser.add_argument('-g', '--gold', dest='gold', help='Specify gold file without conf file.')
	parser.add_argument('-t', '--test', dest='test', help='Specify test file without conf file.')
	parser.add_argument('-d', '--delimeter', dest='delimeter', help='Specify the delimeter', default='/')

	# set defaults
	args = parser.parse_args()

	if not args.conf:
		if not args.gold and args.test:
			sys.stderr.write('Either the conf file or gold and opts file must be specified.')
			parser.print_help()
			sys.exit()
		else:
			gold = args.gold
			test = args.test
			delimeter = args.delimeter
	else:
		c = ConfigFile.ConfigFile(args.conf)
		gold = c['goldfile']
		test = c['testfile']
		delimeter = c.get('delimeter', '/')

		# Default to no tagmap, unless
		# specified in the config file.
		tagmap = c.get('tagmap')

	if gold and test and delimeter:
		slashtags_eval(gold, test, delimeter, tagmap)
	else:
		sys.stderr.write('Arguments missing.')
		sys.exit()
		<|endoftext|>


<commit_before>"""
JSON-Schema implementation of NetJSON DeviceConfiguration
http://netjson.org/rfc.html
"""

schema = {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "additionalProperties": True,
    "definitions": {
        "interface_settings": {
            "type": "object",
            "title": "Interface settings",
            "additionalProperties": True,
            "required": [
                "name",
                "type"
            ],
            "properties": {
                "name": {
                    "type": "string",
                    "maxLength": 15,
                    "pattern": "^[^\\s]*$",
                    "propertyOrder": 0,
                },
                "mac": {
                    "type": "string",
                    "title": "mac address",
                    "propertyOrder": 2,
                },
                "mtu": {
                    "type": "integer",
                    "title": "MTU",
                    "default": 1500,
                    "minimum": 68,
                    "propertyOrder": 3,
                },
                "autostart": {
                    "type": "boolean",
                    "default": True,
                    "format": "checkbox",
                    "propertyOrder": 5,
                },
                "disabled": {
                    "type": "boolean",
                    "default": False,
                    "format": "checkbox",
                    "propertyOrder": 6,
                },
                "addresses": {
                    "type": "array",
                    "title": "Addresses",
                    "uniqueItems": True,
                    "additionalItems": True,
                    "propertyOrder": 20,
                    "items": {
                        "type": "object",
                        "title": "Address",
                        "additionalProperties": True,
                        "required": [
                            "proto",
                            "family"
                        ],
                        "properties": {
                            "proto": {
                                "type": "string",
                                "enum": [
                                    "static",
                                    "dhcp"
                                ],
                                "propertyOrder": 1,
                            },
                            "family": {
                                "type": "string",
                                "enum": [
                                    "ipv4",
                                    "ipv6"
                                ],
                                "propertyOrder": 2,
                            },
                            "address": {
                                "type": "string",
                                "propertyOrder": 3,
                            },
                            "mask": {
                                "type": "integer",
                                "propertyOrder": 4,
                            },
                            "gateway": {
                                "type": "string",
                                "propertyOrder": 5,
                            }
                        }
                    }
                }
            }
        },
        "network_interface": {
            "title": "Network interface",
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": [
                                "ethernet",
                                "virtual",
                                "loopback",
                                "other"
                            ],
                            "propertyOrder": 1,
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"}
            ]
        },
        "wireless_interface": {
            "title": "Wireless interface",
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": ["wireless"],
                            "default": "wireless",
                            "propertyOrder": 1,
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"},
                {
                    "properties": {
                        "wireless": {
                            "type": "object",
                            "title": "Wireless Settings",
                            "additionalProperties": True,
                            "propertyOrder": 8,
                            "required": [
                                "radio",
                                "mode",
                                "ssid"
                            ],
                            "properties": {
                                "radio": {
                                    "type": "string",
                                    "propertyOrder": 1,
                                },
                                "mode": {
                                    "type": "string",
                                    "enum": [
                                        "access_point",
                                        "station",
                                        "adhoc",
                                        "wds",
                                        "monitor",
                                        "802.11s"
                                    ],
                                    "propertyOrder": 2,
                                },
                                "ssid": {
                                    "type": "string",
                                    "maxLength": 32,
                                    "propertyOrder": 3,
                                },
                                "bssid": {
                                    "type": "string",
                                    "propertyOrder": 4,
                                },
                                "hidden": {
                                    "type": "boolean",

                                    "default": False,
                                    "propertyOrder": 5,
                                },
                                "ack_distance": {
                                    "type": "integer",
                                    "minimum": 1,
                                    "propertyOrder": 6,
                                },
                                "rts_threshold": {
                                    "type": "integer",
                                    "minimum": 0,
                                    "maximum": 2346,
                                    "propertyOrder": 7,
                                },
                                "frag_threshold": {
                                    "type": "integer",
                                    "minimum": 0,
                                    "maximum": 2346,
                                    "propertyOrder": 8,
                                },
                                "encryption": {
                                    "type": "object",
                                    "title": "Encryption",
                                    "required": [
                                        "protocol",
                                        "key"
                                    ],
                                    "propertyOrder": 20,
                                    "properties": {
                                        "protocol": {
                                            "type": "string",
                                            "enum": [
                                                "wep_open",
                                                "wep_shared",
                                                "wpa_personal",
                                                "wpa2_personal",
                                                "wpa_personal_mixed",
                                                "wpa_enterprise",
                                                "wpa2_enterprise",
                                                "wpa_enterprise_mixed",
                                                "wps"
                                            ],
                                            "propertyOrder": 1,
                                        },
                                        "disabled": {
                                            "type": "boolean",
                                            "default": False,
                                            "format": "checkbox",
                                            "propertyOrder": 2,
                                        },
                                        "key": {
                                            "type": "string",
                                            "propertyOrder": 3,
                                        },
                                        "ciphers": {
                                            "type": "array",
                                            "propertyOrder": 4,
                                            "items": {
                                                "type": "string"
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            ]
        },
        "bridge_interface": {
            "title": "Bridge interface",
            "required": [
                "bridge_members"
            ],
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": ["bridge"],
                            "propertyOrder": 1
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"},
                {
                    "properties": {
                        "bridge_members": {
                            "type": "array",
                            "title": "Bridge Members",
                            "uniqueItems": True,
                            "propertyOrder": 8,
                            "items": {
                                "title": "bridged interface",
                                "type": "string",
                                "$ref": "#/definitions/interface_settings/properties/name"
                            }
                        }
                    }
                }
            ]
        }
    },
    "properties": {
        "general": {
            "type": "object",
            "title": "General",
            "additionalProperties": True,
            "propertyOrder": 1,
            "properties": {
                "hostname": {
                    "type": "string",
                    "maxLength": 63,
                    "minLength": 1,
                    "propertyOrder": 1,
                },
                "ula_prefix": {
                    "type": "string",
                    "title": "ULA prefix",
                    "propertyOrder": 2,
                },
                "maintainer": {
                    "type": "string",
                    "propertyOrder": 3,
                },
                "description": {
                    "type": "string",
                    "propertyOrder": 4,
                }
            }
        },
        "interfaces": {
            "type": "array",
            "title": "Interfaces",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 2,
            "items": {
                "title": "Interface",
                "oneOf": [
                    {"$ref": "#/definitions/network_interface"},
                    {"$ref": "#/definitions/wireless_interface"},
                    {"$ref": "#/definitions/bridge_interface"}
                ]
            }
        },
        "radios": {
            "type": "array",
            "title": "Radios",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 3,
            "items": {
                "type": "object",
                "title": "Radio",
                "additionalProperties": True,
                "required": [
                    "name",
                    "channel",
                    "channel_width",
                ],
                "properties": {
                    "name": {
                        "type": "string",
                        "propertyOrder": 1,
                    },
                    "phy": {
                        "type": "string",
                        "propertyOrder": 2,
                    },
                    "channel": {
                        "type": "integer",
                        "propertyOrder": 3,
                    },
                    "channel_width": {
                        "type": "integer",
                        "title": "channel width",
                        "propertyOrder": 4,
                    },
                    "tx_power": {
                        "type": "integer",
                        "title": "tx power",
                        "propertyOrder": 5,
                    },
                    "country": {
                        "type": "string",
                        "minLength": 2,
                        "maxLength": 2,
                        "propertyOrder": 5,
                    },
                    "disabled": {
                        "type": "boolean",
                        "default": False,
                        "format": "checkbox",
                        "propertyOrder": 6,
                    }
                }
            }
        },
        "dns_servers": {
            "title": "DNS Configuration",
            "type": "array",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 4,
            "items": {
                "title": "DNS Server",
                "type": "string"
            }
        },
        "dns_search": {
            "title": "DNS Search List",
            "type": "array",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 5,
            "items": {
                "title": "Domain",
                "type": "string"
            }
        },
        "routes": {
            "type": "array",
            "title": "Routes",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 6,
            "items": {
                "type": "object",
                "title": "Route",
                "additionalProperties": True,
                "required": [
                    "device",
                    "destination",
                    "next"
                ],
                "properties": {
                    "device": {
                        "type": "string",
                        "propertyOrder": 1,
                    },
                    "next": {
                        "type": "string",
                        "propertyOrder": 2,
                    },
                    "destination": {
                        "type": "string",
                        "propertyOrder": 3,
                    }
                }
            }
        }
    }
}
<commit_msg>[schema] Added minLength to interface name<commit_after>"""
JSON-Schema implementation of NetJSON DeviceConfiguration
http://netjson.org/rfc.html
"""

schema = {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "additionalProperties": True,
    "definitions": {
        "interface_settings": {
            "type": "object",
            "title": "Interface settings",
            "additionalProperties": True,
            "required": [
                "name",
                "type"
            ],
            "properties": {
                "name": {
                    "type": "string",
                    "maxLength": 15,
                    "minLenght": 2,
                    "pattern": "^[^\\s]*$",
                    "propertyOrder": 0,
                },
                "mac": {
                    "type": "string",
                    "title": "mac address",
                    "propertyOrder": 2,
                },
                "mtu": {
                    "type": "integer",
                    "title": "MTU",
                    "default": 1500,
                    "minimum": 68,
                    "propertyOrder": 3,
                },
                "autostart": {
                    "type": "boolean",
                    "default": True,
                    "format": "checkbox",
                    "propertyOrder": 5,
                },
                "disabled": {
                    "type": "boolean",
                    "default": False,
                    "format": "checkbox",
                    "propertyOrder": 6,
                },
                "addresses": {
                    "type": "array",
                    "title": "Addresses",
                    "uniqueItems": True,
                    "additionalItems": True,
                    "propertyOrder": 20,
                    "items": {
                        "type": "object",
                        "title": "Address",
                        "additionalProperties": True,
                        "required": [
                            "proto",
                            "family"
                        ],
                        "properties": {
                            "proto": {
                                "type": "string",
                                "enum": [
                                    "static",
                                    "dhcp"
                                ],
                                "propertyOrder": 1,
                            },
                            "family": {
                                "type": "string",
                                "enum": [
                                    "ipv4",
                                    "ipv6"
                                ],
                                "propertyOrder": 2,
                            },
                            "address": {
                                "type": "string",
                                "propertyOrder": 3,
                            },
                            "mask": {
                                "type": "integer",
                                "propertyOrder": 4,
                            },
                            "gateway": {
                                "type": "string",
                                "propertyOrder": 5,
                            }
                        }
                    }
                }
            }
        },
        "network_interface": {
            "title": "Network interface",
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": [
                                "ethernet",
                                "virtual",
                                "loopback",
                                "other"
                            ],
                            "propertyOrder": 1,
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"}
            ]
        },
        "wireless_interface": {
            "title": "Wireless interface",
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": ["wireless"],
                            "default": "wireless",
                            "propertyOrder": 1,
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"},
                {
                    "properties": {
                        "wireless": {
                            "type": "object",
                            "title": "Wireless Settings",
                            "additionalProperties": True,
                            "propertyOrder": 8,
                            "required": [
                                "radio",
                                "mode",
                                "ssid"
                            ],
                            "properties": {
                                "radio": {
                                    "type": "string",
                                    "propertyOrder": 1,
                                },
                                "mode": {
                                    "type": "string",
                                    "enum": [
                                        "access_point",
                                        "station",
                                        "adhoc",
                                        "wds",
                                        "monitor",
                                        "802.11s"
                                    ],
                                    "propertyOrder": 2,
                                },
                                "ssid": {
                                    "type": "string",
                                    "maxLength": 32,
                                    "propertyOrder": 3,
                                },
                                "bssid": {
                                    "type": "string",
                                    "propertyOrder": 4,
                                },
                                "hidden": {
                                    "type": "boolean",

                                    "default": False,
                                    "propertyOrder": 5,
                                },
                                "ack_distance": {
                                    "type": "integer",
                                    "minimum": 1,
                                    "propertyOrder": 6,
                                },
                                "rts_threshold": {
                                    "type": "integer",
                                    "minimum": 0,
                                    "maximum": 2346,
                                    "propertyOrder": 7,
                                },
                                "frag_threshold": {
                                    "type": "integer",
                                    "minimum": 0,
                                    "maximum": 2346,
                                    "propertyOrder": 8,
                                },
                                "encryption": {
                                    "type": "object",
                                    "title": "Encryption",
                                    "required": [
                                        "protocol",
                                        "key"
                                    ],
                                    "propertyOrder": 20,
                                    "properties": {
                                        "protocol": {
                                            "type": "string",
                                            "enum": [
                                                "wep_open",
                                                "wep_shared",
                                                "wpa_personal",
                                                "wpa2_personal",
                                                "wpa_personal_mixed",
                                                "wpa_enterprise",
                                                "wpa2_enterprise",
                                                "wpa_enterprise_mixed",
                                                "wps"
                                            ],
                                            "propertyOrder": 1,
                                        },
                                        "disabled": {
                                            "type": "boolean",
                                            "default": False,
                                            "format": "checkbox",
                                            "propertyOrder": 2,
                                        },
                                        "key": {
                                            "type": "string",
                                            "propertyOrder": 3,
                                        },
                                        "ciphers": {
                                            "type": "array",
                                            "propertyOrder": 4,
                                            "items": {
                                                "type": "string"
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            ]
        },
        "bridge_interface": {
            "title": "Bridge interface",
            "required": [
                "bridge_members"
            ],
            "allOf": [
                {
                    "properties": {
                        "type": {
                            "type": "string",
                            "enum": ["bridge"],
                            "propertyOrder": 1
                        }
                    }
                },
                {"$ref": "#/definitions/interface_settings"},
                {
                    "properties": {
                        "bridge_members": {
                            "type": "array",
                            "title": "Bridge Members",
                            "uniqueItems": True,
                            "propertyOrder": 8,
                            "items": {
                                "title": "bridged interface",
                                "type": "string",
                                "$ref": "#/definitions/interface_settings/properties/name"
                            }
                        }
                    }
                }
            ]
        }
    },
    "properties": {
        "general": {
            "type": "object",
            "title": "General",
            "additionalProperties": True,
            "propertyOrder": 1,
            "properties": {
                "hostname": {
                    "type": "string",
                    "maxLength": 63,
                    "minLength": 1,
                    "propertyOrder": 1,
                },
                "ula_prefix": {
                    "type": "string",
                    "title": "ULA prefix",
                    "propertyOrder": 2,
                },
                "maintainer": {
                    "type": "string",
                    "propertyOrder": 3,
                },
                "description": {
                    "type": "string",
                    "propertyOrder": 4,
                }
            }
        },
        "interfaces": {
            "type": "array",
            "title": "Interfaces",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 2,
            "items": {
                "title": "Interface",
                "oneOf": [
                    {"$ref": "#/definitions/network_interface"},
                    {"$ref": "#/definitions/wireless_interface"},
                    {"$ref": "#/definitions/bridge_interface"}
                ]
            }
        },
        "radios": {
            "type": "array",
            "title": "Radios",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 3,
            "items": {
                "type": "object",
                "title": "Radio",
                "additionalProperties": True,
                "required": [
                    "name",
                    "channel",
                    "channel_width",
                ],
                "properties": {
                    "name": {
                        "type": "string",
                        "propertyOrder": 1,
                    },
                    "phy": {
                        "type": "string",
                        "propertyOrder": 2,
                    },
                    "channel": {
                        "type": "integer",
                        "propertyOrder": 3,
                    },
                    "channel_width": {
                        "type": "integer",
                        "title": "channel width",
                        "propertyOrder": 4,
                    },
                    "tx_power": {
                        "type": "integer",
                        "title": "tx power",
                        "propertyOrder": 5,
                    },
                    "country": {
                        "type": "string",
                        "minLength": 2,
                        "maxLength": 2,
                        "propertyOrder": 5,
                    },
                    "disabled": {
                        "type": "boolean",
                        "default": False,
                        "format": "checkbox",
                        "propertyOrder": 6,
                    }
                }
            }
        },
        "dns_servers": {
            "title": "DNS Configuration",
            "type": "array",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 4,
            "items": {
                "title": "DNS Server",
                "type": "string"
            }
        },
        "dns_search": {
            "title": "DNS Search List",
            "type": "array",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 5,
            "items": {
                "title": "Domain",
                "type": "string"
            }
        },
        "routes": {
            "type": "array",
            "title": "Routes",
            "uniqueItems": True,
            "additionalItems": True,
            "propertyOrder": 6,
            "items": {
                "type": "object",
                "title": "Route",
                "additionalProperties": True,
                "required": [
                    "device",
                    "destination",
                    "next"
                ],
                "properties": {
                    "device": {
                        "type": "string",
                        "propertyOrder": 1,
                    },
                    "next": {
                        "type": "string",
                        "propertyOrder": 2,
                    },
                    "destination": {
                        "type": "string",
                        "propertyOrder": 3,
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>"""
Functions to plot and extract CSD info from LFP data
Contributors: Erica Y Griffith, Sam Neymotin, Salvador Dura-Bernal
"""

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import

# THIRD PARTY IMPORTS
import numpy as np
from future import standard_library
standard_library.install_aliases()

## LOCAL APPLICATION IMPORTS
from .filter import lowpass,bandpass
from .utils import exception, _saveFigData, _showFigure

############################################
######## FUNCTIONS USED IN getCSD() ########
############################################

# Bandpass filter
def getbandpass (lfps,sampr,minf=0.05,maxf=300):   # lfps should be a list or numpy array of LFPs arranged spatially in a column
  datband = []
  for i in range(len(lfps[0])): datband.append(bandpass(lfps[:,i],minf,maxf,df=sampr,zerophase=True))
  datband = np.array(datband)
  return datband

# Vaknin correction for CSD analysis (MS implementation)
def Vaknin (x): ## Allows CSD to be performed on all N contacts instead of N-2 contacts (See Vaknin et al (1988) for more details)
  # Preallocate array with 2 more rows than input array
  x_new = np.zeros((x.shape[0]+2, x.shape[1]))
  # print(x_new.shape)
  # Duplicate first and last row of x into first and last row of x_new
  x_new[0, :] = x[0, :]
  x_new[-1, :] = x[-1, :]
  # Duplicate all of x into middle rows of x_neww
  x_new[1:-1, :] = x
  return x_new

# REMOVE MEAN
def removemean (x, ax=1):
  mean = np.mean(x, axis=ax, keepdims=True)
  x -= mean
  #print(np.mean(x, axis=ax, keepdims=True))

################################################
######### GET CSD VALUES FROM LFP DATA #########
################################################
@exception
def getCSD (sampr=None,timeRange=None,spacing_um=100.0,minf=0.05,maxf=300,norm=True,vaknin=False):
  """ Extracts CSD values from simulated LFP data

      Parameters
      ----------
      sampr : float
        Sampling rate for data recording.
        **Default:**
        ``None`` uses cfg.recordStep

      timeRange : list [start, stop]
        Time range to plot.
        **Default:**
        ``None`` plots entire time range

      spacing_um : float
        Electrode's contact spacing in units of microns <-- VERTICALLY, I ASSUME?
        **Default:** ``100.0``

      minf : float
        Minimum frequency for bandpass filter (Hz).
        **Default:** ``0.05``

      maxf : float
        Maximum frequency cutoff for bandpass filter (Hz).
        **Default:** ``300``

      norm : bool
        Needs documentation. <---- ??
        **Default:**
        ``True``

      vaknin : bool
        Needs documentation. <---- ??
        **Default**
        ``False``
  """

  from .. import sim

  ## SET DEFAULT ARGUMENT / PARAMETER VALUES
  if timeRange is None:                 # Specify the time range of relevant LFP data
    timeRange = [0,sim.cfg.duration]    # This makes the timeRange equal to the entire sim duration

  if sampr is None:
    sampr = sim.cfg.recordStep          # Sampling rate of data recording during the simulation

  spacing_mm = spacing_um/1000          # Spacing between electrodes --> convert from micron to mm

  ## Check if LFP was recorded during the simulation
  sim_data_categories = sim.allSimData.keys()

  if 'LFP' in sim_data_categories:
    # Get LFP data
    lfp_data = np.array(sim.allSimData['LFP'])[int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep),:] # from lfp.py, line 200; array splicing

    # Bandpass filter the LFP data with getbandpass() fx defined above
    datband = getbandpass(lfp_data,sampr,minf,maxf)

    # Take CSD along smaller dimension
    if datband.shape[0] > datband.shape[1]:
      ax = 1
    else:
      ax = 0

    # VAKNIN CORRECTION
    if vaknin:
      datband = Vaknin(datband)

    # NORM <-- ASKING SAM MORE ABOUT THIS
    if norm:
      removemean(datband,ax=ax)

    # now each column (or row) is an electrode -- take CSD along electrodes
    CSD_data = -np.diff(datband,n=2,axis=ax)/spacing_mm**2  ## CSD_data should be in mV/mm**2, assuming that LFP data is in mV.

    # Add CSD values to sim.allSimData for access outside of this function or script
    sim.allSimData['CSD'] = CSD_data

  elif 'LFP' not in sim_data:
    print('!! WARNING: NO LFP DATA !! Need to re-run simulation with cfg.recordLFP enabled')
    CSD_data = []

  # returns CSD in units of mV/mm**2 (assuming lfps are in mV)
  return CSD_data

################################
######### PLOTTING CSD #########
################################

def plotCSD(timeRange=None, saveData=None, saveFig=None, showFig=True):
  """ Plots CSD values extracted from simulated LFP data

      Parameters
      ----------
      timeRange : list [start, stop]
        Time range to plot.
        **Default:**
        ``None`` plots entire time range

      saveData : bool or str
        Whether and where to save the data used to generate the plot.
        **Default:** ``False``
        **Options:** ``True`` autosaves the data
        ``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.pkl'`` and ``'.json'``

     saveFig : bool or str
        Whether and where to save the figure.
        **Default:** ``False``
        **Options:** ``True`` autosaves the figure,
        ``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.png'``, ``'.jpg'``, ``'.eps'``, and ``'.tiff'``

      showFig : bool
        Shows the figure if ``True``.
        **Default:** ``True``

  """
  print('Plotting CSD... ') # NO PLOT YET
  sim_data = sim.allSimData.keys()

  if 'CSD' in sim_data:
    print('CSD values have already been extracted from LFP by getCSD()')
  else:
    sim.analysis.getCSD() # WHAT ABOUT ARGS?

# NOTE ON COLORS:
# # when drawing CSD make sure that negative values (depolarizing intracellular current) drawn in red,
# # and positive values (hyperpolarizing intracellular current) drawn in blue

##################################################
################ UNUSED FUNCTIONS ################
##################################################

## These are from Sam Neymotin; https://github.com/NathanKlineInstitute/OEvent/blob/master/csd.py

### LOWPASS FILTER ###

# # lowpass filter the items in lfps. lfps is a list or numpy array of LFPs arranged spatially by column
# def getlowpass (lfps,sampr,maxf):
#   datlow = []
#   for i in range(len(lfps[0])): datlow.append(lowpass(lfps[:,i],maxf,df=sampr,zerophase=True))
#   datlow = np.array(datlow)
#   return datlow

### BIPOLAR WAVEFORMS ###

# # get bipolar waveforms - first do a lowpass filter. lfps is a list or numpy array of LFPs arranged spatially by column
# # spacing_um is electrode's contact spacing in units of micron
# # returns bipolar signal in units of mV/mm (assuming lfps are in mV)
# def getBipolar (lfps,sampr,spacing_um=100.0,minf=0.05,maxf=300,norm=True,vaknin=False):
#   datband = getbandpass(lfps,sampr,minf,maxf)
#   if datband.shape[0] > datband.shape[1]: # take CSD along smaller dimension
#     ax = 1
#   else:
#     ax = 0
#   if vaknin: datband = Vaknin(datband)
#   if norm: removemean(datband,ax=ax)
#   # can change to run Vaknin on bandpass filtered LFPs before calculating bipolar waveforms, that
#   # way would have same number of channels in bipolar and LFP (but not critical, and would take more RAM);
#   # also might want to subtract mean of each channel before calculating the diff ?
#   spacing_mm = spacing_um/1000 # spacing in mm
#   bipolar = -np.diff(datband,n=1,axis=ax)/spacing_mm # now each column (or row) is an electrode -- bipolar along electrodes
#   return bipolar

### MULTI-UNIT ACTIVITY ###

# # get MUA - first do a bandpass filter then rectify.
# #  lfps is a list or numpy array of LFPs arranged spatially by column
# def getMUA (lfps,sampr,minf=300,maxf=5000):
#   datband = getbandpass(lfps,sampr,minf,maxf)
#   return np.abs(datband)<commit_msg>adding some lines for storing CSD values and timeRange<commit_after>"""
Functions to plot and extract CSD info from LFP data
Contributors: Erica Y Griffith, Sam Neymotin, Salvador Dura-Bernal
"""

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import

# THIRD PARTY IMPORTS
import numpy as np
from future import standard_library
standard_library.install_aliases()

## LOCAL APPLICATION IMPORTS
from .filter import lowpass,bandpass
from .utils import exception, _saveFigData, _showFigure

############################################
######## FUNCTIONS USED IN getCSD() ########
############################################

# Bandpass filter
def getbandpass (lfps,sampr,minf=0.05,maxf=300):   # lfps should be a list or numpy array of LFPs arranged spatially in a column
  datband = []
  for i in range(len(lfps[0])): datband.append(bandpass(lfps[:,i],minf,maxf,df=sampr,zerophase=True))
  datband = np.array(datband)
  return datband

# Vaknin correction for CSD analysis (MS implementation)
def Vaknin (x): ## Allows CSD to be performed on all N contacts instead of N-2 contacts (See Vaknin et al (1988) for more details)
  # Preallocate array with 2 more rows than input array
  x_new = np.zeros((x.shape[0]+2, x.shape[1]))
  # print(x_new.shape)
  # Duplicate first and last row of x into first and last row of x_new
  x_new[0, :] = x[0, :]
  x_new[-1, :] = x[-1, :]
  # Duplicate all of x into middle rows of x_neww
  x_new[1:-1, :] = x
  return x_new

# REMOVE MEAN
def removemean (x, ax=1):
  mean = np.mean(x, axis=ax, keepdims=True)
  x -= mean
  #print(np.mean(x, axis=ax, keepdims=True))

################################################
######### GET CSD VALUES FROM LFP DATA #########
################################################
@exception
def getCSD (sampr=None,timeRange=None,spacing_um=100.0,minf=0.05,maxf=300,norm=True,vaknin=False):
  """ Extracts CSD values from simulated LFP data

      Parameters
      ----------
      sampr : float
        Sampling rate for data recording.
        **Default:**
        ``None`` uses cfg.recordStep

      timeRange : list [start, stop]
        Time range to plot.
        **Default:**
        ``None`` plots entire time range

      spacing_um : float
        Electrode's contact spacing in units of microns <-- VERTICALLY, I ASSUME?
        **Default:** ``100.0``

      minf : float
        Minimum frequency for bandpass filter (Hz).
        **Default:** ``0.05``

      maxf : float
        Maximum frequency cutoff for bandpass filter (Hz).
        **Default:** ``300``

      norm : bool
        Needs documentation. <---- ??
        **Default:**
        ``True``

      vaknin : bool
        Needs documentation. <---- ??
        **Default**
        ``False``
  """

  from .. import sim

  ## SET DEFAULT ARGUMENT / PARAMETER VALUES
  if timeRange is None:                 # Specify the time range of relevant LFP data
    timeRange = [0,sim.cfg.duration]    # This makes the timeRange equal to the entire sim duration

  if sampr is None:
    sampr = sim.cfg.recordStep          # Sampling rate of data recording during the simulation

  spacing_mm = spacing_um/1000          # Spacing between electrodes --> convert from micron to mm

  ## Check if LFP was recorded during the simulation
  sim_data_categories = sim.allSimData.keys()

  if 'LFP' in sim_data_categories:
    # Get LFP data
    lfp_data = np.array(sim.allSimData['LFP'])[int(timeRange[0]/sim.cfg.recordStep):int(timeRange[1]/sim.cfg.recordStep),:] # from lfp.py, line 200; array splicing

    # Bandpass filter the LFP data with getbandpass() fx defined above
    datband = getbandpass(lfp_data,sampr,minf,maxf)

    # Take CSD along smaller dimension
    if datband.shape[0] > datband.shape[1]:
      ax = 1
    else:
      ax = 0

    # VAKNIN CORRECTION
    if vaknin:
      datband = Vaknin(datband)

    # NORM <-- ASKING SAM MORE ABOUT THIS
    if norm:
      removemean(datband,ax=ax)

    # now each column (or row) is an electrode -- take CSD along electrodes
    CSD_data = -np.diff(datband,n=2,axis=ax)/spacing_mm**2  ## CSD_data should be in mV/mm**2, assuming that LFP data is in mV.

    # Add CSD values to sim.allSimData for access outside of this function or script
    sim.allSimData['CSD'] = CSD_data

  elif 'LFP' not in sim_data:
    print('!! WARNING: NO LFP DATA !! Need to re-run simulation with cfg.recordLFP enabled')
    CSD_data = []

  # returns CSD in units of mV/mm**2 (assuming lfps are in mV)
  return CSD_data

################################
######### PLOTTING CSD #########
################################

def plotCSD(timeRange=None, saveData=None, saveFig=None, showFig=True):
  """ Plots CSD values extracted from simulated LFP data

      Parameters
      ----------
      timeRange : list [start, stop]
        Time range to plot.
        **Default:**
        ``None`` plots entire time range

      saveData : bool or str
        Whether and where to save the data used to generate the plot.
        **Default:** ``False``
        **Options:** ``True`` autosaves the data
        ``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.pkl'`` and ``'.json'``

     saveFig : bool or str
        Whether and where to save the figure.
        **Default:** ``False``
        **Options:** ``True`` autosaves the figure,
        ``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.png'``, ``'.jpg'``, ``'.eps'``, and ``'.tiff'``

      showFig : bool
        Shows the figure if ``True``.
        **Default:** ``True``

  """
  from .. import sim

  print('Plotting CSD... ') # NO PLOT YET

  ## CHECK IF CSD VALUES HAVE ALREADY BEEN EXTRACTED FROM LFP
  sim_data = sim.allSimData.keys()

  ## STORE CSD DATA
  if 'CSD' in sim_data:
    CSD_data = sim.allSimData['CSD']
  elif 'CSD' not in sim_data:
    print('NEED TO GET CSD VALUES FROM LFP DATA -- run sim.analysis.getCSD()')
    sim.analysis.getCSD()   # WHAT ABOUT ARGS? ANY NEEDED?
    CSD_data = sim.allSimData['CSD']

  ## time range
  if timeRange is None:
    timeRange = [0, sim.cfg.duration]   # default time range is entire length of the sim

# NOTE ON COLORS:
# # when drawing CSD make sure that negative values (depolarizing intracellular current) drawn in red,
# # and positive values (hyperpolarizing intracellular current) drawn in blue

##################################################
################ UNUSED FUNCTIONS ################
##################################################

## These are from Sam Neymotin; https://github.com/NathanKlineInstitute/OEvent/blob/master/csd.py

### LOWPASS FILTER ###

# # lowpass filter the items in lfps. lfps is a list or numpy array of LFPs arranged spatially by column
# def getlowpass (lfps,sampr,maxf):
#   datlow = []
#   for i in range(len(lfps[0])): datlow.append(lowpass(lfps[:,i],maxf,df=sampr,zerophase=True))
#   datlow = np.array(datlow)
#   return datlow

### BIPOLAR WAVEFORMS ###

# # get bipolar waveforms - first do a lowpass filter. lfps is a list or numpy array of LFPs arranged spatially by column
# # spacing_um is electrode's contact spacing in units of micron
# # returns bipolar signal in units of mV/mm (assuming lfps are in mV)
# def getBipolar (lfps,sampr,spacing_um=100.0,minf=0.05,maxf=300,norm=True,vaknin=False):
#   datband = getbandpass(lfps,sampr,minf,maxf)
#   if datband.shape[0] > datband.shape[1]: # take CSD along smaller dimension
#     ax = 1
#   else:
#     ax = 0
#   if vaknin: datband = Vaknin(datband)
#   if norm: removemean(datband,ax=ax)
#   # can change to run Vaknin on bandpass filtered LFPs before calculating bipolar waveforms, that
#   # way would have same number of channels in bipolar and LFP (but not critical, and would take more RAM);
#   # also might want to subtract mean of each channel before calculating the diff ?
#   spacing_mm = spacing_um/1000 # spacing in mm
#   bipolar = -np.diff(datband,n=1,axis=ax)/spacing_mm # now each column (or row) is an electrode -- bipolar along electrodes
#   return bipolar

### MULTI-UNIT ACTIVITY ###

# # get MUA - first do a bandpass filter then rectify.
# #  lfps is a list or numpy array of LFPs arranged spatially by column
# def getMUA (lfps,sampr,minf=300,maxf=5000):
#   datband = getbandpass(lfps,sampr,minf,maxf)
#   return np.abs(datband)<|endoftext|>