<commit_before>#!/usr/bin/python
#
#
##
# Import PLC records into the Geni database. It is indended that this tool be
# run once to create Geni records that reflect the current state of the
# planetlab database.
#
# The import tool assumes that the existing PLC hierarchy should all be part
# of "planetlab.us" (see the root_auth and level1_auth variables below).
#
# Public keys are extracted from the users' SSH keys automatically and used to
# create GIDs. This is relatively experimental as a custom tool had to be
# written to perform conversion from SSH to OpenSSL format. It only supports
# RSA keys at this time, not DSA keys.
##

import getopt
import sys
import tempfile

from geni.util.cert import *
from geni.util.trustedroot import *
from geni.util.hierarchy import *
from geni.util.record import *
from geni.util.genitable import *
from geni.util.misc import *
from geni.util.config import *

# get PL account settings from config module
pl_auth = get_pl_auth()

# connect to planetlab
if "Url" in pl_auth:
    from geni.util import remoteshell
    shell = remoteshell.RemoteShell()
else:
    import PLC.Shell
    shell = PLC.Shell.Shell(globals = globals())

##
# Two authorities are specified: the root authority and the level1 authority.

#root_auth = "plc"
#level1_auth = None

#root_auth = "planetlab"
#level1_auth = "planetlab.us"
config = Config()

root_auth = config.GENI_REGISTRY_ROOT_AUTH
level1_auth = config.GENI_REGISTRY_LEVEL1_AUTH
if not level1_auth or level1_auth in ['']:
    level1_auth = None

keyconvert = 'keyconvert'
loaded = False
default_path = "/usr/share/keyconvert/" + keyconvert
cwd = os.path.dirname(os.path.abspath(__file__))
alt_path = os.sep.join(cwd.split(os.sep)[:-1] + ['keyconvert', 'keyconvert'])
geni_path = config.GENI_BASE_DIR + os.sep + "keyconvert/keyconvert"
files = [default_path, alt_path, geni_path]
for path in files:
    print os.path.isfile(path), path
    if os.path.isfile(path):
        keyconvert_fn = path
        loaded = True
        break

if not loaded:
    raise Exception, "Could not find config in " + ", ".join(files)

keyconvert_fn = config.GENI_BASE_DIR + os.sep + "keyconvert/keyconvert"

def un_unicode(str):
   if isinstance(str, unicode):
       return str.encode("ascii", "ignore")
   else:
       return str

def cleanup_string(str):
    # pgsql has a fit with strings that have high ascii in them, so filter it
    # out when generating the hrns.
    tmp = ""
    for c in str:
        if ord(c) < 128:
            tmp = tmp + c
    str = tmp

    str = un_unicode(str)
    str = str.replace(" ", "_")
    str = str.replace(".", "_")
    str = str.replace("(", "_")
    str = str.replace("'", "_")
    str = str.replace(")", "_")
    str = str.replace('"', "_")
    return str

def process_options():
   global hrn

   (options, args) = getopt.getopt(sys.argv[1:], '', [])
   for opt in options:
       name = opt[0]
       val = opt[1]

def connect_shell():
    global pl_auth, shell

    # get PL account settings from config module
    pl_auth = get_pl_auth()

    # connect to planetlab
    if "Url" in pl_auth:
        from geni.util import remoteshell
        shell = remoteshell.RemoteShell()
    else:
        import PLC.Shell
        shell = PLC.Shell.Shell(globals = globals())

    return shell

def get_auth_table(auth_name):
    AuthHierarchy = Hierarchy()
    auth_info = AuthHierarchy.get_auth_info(auth_name)

    table = GeniTable(hrn=auth_name,
                      cninfo=auth_info.get_dbinfo())

    # if the table doesn't exist, then it means we haven't put any records
    # into this authority yet.

    if not table.exists():
        report.trace("Import: creating table for authority " + auth_name)
        table.create()

    return table

def get_pl_pubkey(key_id):
    keys = shell.GetKeys(pl_auth, [key_id])
    if keys:
        key_str = keys[0]['key']

        if "ssh-dss" in key_str:
            print "XXX: DSA key encountered, ignoring"
            return None

        # generate temporary files to hold the keys
        (ssh_f, ssh_fn) = tempfile.mkstemp()
        ssl_fn = tempfile.mktemp()

        os.write(ssh_f, key_str)
        os.close(ssh_f)

        if not os.path.exists(keyconvert_fn):
            report.trace("  keyconvert utility " + str(keyconvert_fn) + " does not exist");
            sys.exit(-1)

        cmd = keyconvert_fn + " " + ssh_fn + " " + ssl_fn
        print cmd
        os.system(cmd)

        # this check leaves the temporary file containing the public key so
        # that it can be expected to see why it failed.
        # TODO: for production, cleanup the temporary files
        if not os.path.exists(ssl_fn):
            report.trace("  failed to convert key from " + ssh_fn + " to " + ssl_fn)
            return None

        k = Keypair()
        try:
            k.load_pubkey_from_file(ssl_fn)
        except:
            print "XXX: Error while converting key: ", key_str
            k = None

        # remove the temporary files
        os.remove(ssh_fn)
        os.remove(ssl_fn)

        return k
    else:
        return None

def person_to_hrn(parent_hrn, person):
    # the old way - Lastname_Firstname
    #personname = person['last_name'] + "_" + person['first_name']

    # the new way - use email address up to the "@"
    personname = person['email'].split("@")[0]

    personname = cleanup_string(personname)

    hrn = parent_hrn + "." + personname
    return hrn

def import_person(parent_hrn, person):
    AuthHierarchy = Hierarchy()
    hrn = person_to_hrn(parent_hrn, person)

    # ASN.1 will have problems with hrn's longer than 64 characters
    if len(hrn) > 64:
        hrn = hrn[:64]

    report.trace("Import: importing person " + hrn)

    table = get_auth_table(parent_hrn)

    person_record = table.resolve("user", hrn)
    if not person_record:
        key_ids = []
        if 'key_ids' in person:
            key_ids = person["key_ids"]

        if key_ids:
            # get the user's private key from the SSH keys they have uploaded
            # to planetlab
            pkey = get_pl_pubkey(key_ids[0])
        else:
            # the user has no keys
            report.trace("   person " + hrn + " does not have a PL public key")
            pkey = None

        # if a key is unavailable, then we still need to put something in the
        # user's GID. So make one up.
        if not pkey:
            pkey = Keypair(create=True)

        person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
        report.trace("  inserting user record for " + hrn)
        table.insert(person_record)
    else:
        key_ids = person["key_ids"]

    if key_ids:
        pkey = get_pl_pubkey(key_ids[0])
        person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
        report.trace("  updating user record for " + hrn)
        table.update(person_record)

def import_slice(parent_hrn, slice):
    AuthHierarchy = Hierarchy()
    slicename = slice['name'].split("_",1)[-1]
    slicename = cleanup_string(slicename)

    if not slicename:
        report.error("Import_Slice: failed to parse slice name " + slice['name'])
        return

    hrn = parent_hrn + "." + slicename
    report.trace("Import: importing slice " + hrn)

    table = get_auth_table(parent_hrn)

    slice_record = table.resolve("slice", hrn)
    if not slice_record:
        pkey = Keypair(create=True)
        slice_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        slice_record = GeniRecord(name=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'])
        report.trace("  inserting slice record for " + hrn)
        table.insert(slice_record)

def import_node(parent_hrn, node):
    AuthHierarchy = Hierarchy()
    nodename = node['hostname'].split(".")[0]
    nodename = cleanup_string(nodename)

    if not nodename:
        report.error("Import_node: failed to parse node name " + node['hostname'])
        return

    hrn = parent_hrn + "." + nodename

    # ASN.1 will have problems with hrn's longer than 64 characters
    if len(hrn) > 64:
        hrn = hrn[:64]

    report.trace("Import: importing node " + hrn)

    table = get_auth_table(parent_hrn)

    node_record = table.resolve("node", hrn)
    if not node_record:
        pkey = Keypair(create=True)
        node_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        node_record = GeniRecord(name=hrn, gid=node_gid, type="node", pointer=node['node_id'])
        report.trace("  inserting node record for " + hrn)
        table.insert(node_record)

def import_site(parent_hrn, site):
    AuthHierarchy = Hierarchy()
    sitename = site['login_base']
    sitename = cleanup_string(sitename)

    hrn = parent_hrn + "." + sitename

    report.trace("Import_Site: importing site " + hrn)

    # create the authority
    if not AuthHierarchy.auth_exists(hrn):
        AuthHierarchy.create_auth(hrn)

    auth_info = AuthHierarchy.get_auth_info(hrn)

    table = get_auth_table(parent_hrn)

    auth_record = table.resolve("authority", hrn)
    if not auth_record:
        auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id'])
        report.trace("  inserting authority record for " + hrn)
        table.insert(auth_record)

    if 'person_ids' in site:
        for person_id in site['person_ids']:
            persons = shell.GetPersons(pl_auth, [person_id])
            if persons:
                try:
                    import_person(hrn, persons[0])
                except:
                    report.trace("Failed to import: %s" % persons[0])
    if 'slice_ids' in site:
        for slice_id in site['slice_ids']:
            slices = shell.GetSlices(pl_auth, [slice_id])
            if slices:
                try:
                    import_slice(hrn, slices[0])
                except:
                    report.trace("Failed to import: %s" % slices[0])
    if 'node_ids' in site:
        for node_id in site['node_ids']:
            nodes = shell.GetNodes(pl_auth, [node_id])
            if nodes:
                try:
                    import_node(hrn, nodes[0])
                except:
                    report.trace("Failed to import: %s" % nodes[0])

def create_top_level_auth_records(hrn):
    parent_hrn = get_authority(hrn)
    print hrn, ":", parent_hrn
    if not parent_hrn:
        parent_hrn = hrn
    auth_info = AuthHierarchy.get_auth_info(parent_hrn)
    table = get_auth_table(parent_hrn)

    auth_record = table.resolve("authority", hrn)
    if not auth_record:
        auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1)
        report.trace("  inserting authority record for " + hrn)
        table.insert(auth_record)

def main():
    global AuthHierarchy
    global TrustedRoots

    process_options()

    print "Base Directory: ", config.GENI_BASE_DIR

    AuthHierarchy = Hierarchy()
    TrustedRoots = TrustedRootList()

    print "Import: creating top level authorities"

    if not AuthHierarchy.auth_exists(root_auth):
        AuthHierarchy.create_auth(root_auth)

    create_top_level_auth_records(root_auth)
    if level1_auth:
        if not AuthHierarchy.auth_exists(level1_auth):
            AuthHierarchy.create_auth(level1_auth)
        create_top_level_auth_records(level1_auth)
        import_auth = level1_auth
    else:
        import_auth = root_auth

    print "Import: adding", root_auth, "to trusted list"
    root = AuthHierarchy.get_auth_info(root_auth)
    TrustedRoots.add_gid(root.get_gid_object())

    connect_shell()

    sites = shell.GetSites(pl_auth)
    for site in sites:
        import_site(import_auth, site)

if __name__ == "__main__":
    main()
<commit_msg>removed debugging output<commit_after>#!/usr/bin/python
#
#
##
# Import PLC records into the Geni database. It is indended that this tool be
# run once to create Geni records that reflect the current state of the
# planetlab database.
#
# The import tool assumes that the existing PLC hierarchy should all be part
# of "planetlab.us" (see the root_auth and level1_auth variables below).
#
# Public keys are extracted from the users' SSH keys automatically and used to
# create GIDs. This is relatively experimental as a custom tool had to be
# written to perform conversion from SSH to OpenSSL format. It only supports
# RSA keys at this time, not DSA keys.
##

import getopt
import sys
import tempfile

from geni.util.cert import *
from geni.util.trustedroot import *
from geni.util.hierarchy import *
from geni.util.record import *
from geni.util.genitable import *
from geni.util.misc import *
from geni.util.config import *

# get PL account settings from config module
pl_auth = get_pl_auth()

# connect to planetlab
if "Url" in pl_auth:
    from geni.util import remoteshell
    shell = remoteshell.RemoteShell()
else:
    import PLC.Shell
    shell = PLC.Shell.Shell(globals = globals())

##
# Two authorities are specified: the root authority and the level1 authority.

#root_auth = "plc"
#level1_auth = None

#root_auth = "planetlab"
#level1_auth = "planetlab.us"
config = Config()

root_auth = config.GENI_REGISTRY_ROOT_AUTH
level1_auth = config.GENI_REGISTRY_LEVEL1_AUTH
if not level1_auth or level1_auth in ['']:
    level1_auth = None

keyconvert = 'keyconvert'
loaded = False
default_path = "/usr/share/keyconvert/" + keyconvert
cwd = os.path.dirname(os.path.abspath(__file__))
alt_path = os.sep.join(cwd.split(os.sep)[:-1] + ['keyconvert', 'keyconvert'])
geni_path = config.GENI_BASE_DIR + os.sep + "keyconvert/keyconvert"
files = [default_path, alt_path, geni_path]
for path in files:
    if os.path.isfile(path):
        keyconvert_fn = path
        loaded = True
        break

if not loaded:
    raise Exception, "Could not find config in " + ", ".join(files)

keyconvert_fn = config.GENI_BASE_DIR + os.sep + "keyconvert/keyconvert"

def un_unicode(str):
   if isinstance(str, unicode):
       return str.encode("ascii", "ignore")
   else:
       return str

def cleanup_string(str):
    # pgsql has a fit with strings that have high ascii in them, so filter it
    # out when generating the hrns.
    tmp = ""
    for c in str:
        if ord(c) < 128:
            tmp = tmp + c
    str = tmp

    str = un_unicode(str)
    str = str.replace(" ", "_")
    str = str.replace(".", "_")
    str = str.replace("(", "_")
    str = str.replace("'", "_")
    str = str.replace(")", "_")
    str = str.replace('"', "_")
    return str

def process_options():
   global hrn

   (options, args) = getopt.getopt(sys.argv[1:], '', [])
   for opt in options:
       name = opt[0]
       val = opt[1]

def connect_shell():
    global pl_auth, shell

    # get PL account settings from config module
    pl_auth = get_pl_auth()

    # connect to planetlab
    if "Url" in pl_auth:
        from geni.util import remoteshell
        shell = remoteshell.RemoteShell()
    else:
        import PLC.Shell
        shell = PLC.Shell.Shell(globals = globals())

    return shell

def get_auth_table(auth_name):
    AuthHierarchy = Hierarchy()
    auth_info = AuthHierarchy.get_auth_info(auth_name)

    table = GeniTable(hrn=auth_name,
                      cninfo=auth_info.get_dbinfo())

    # if the table doesn't exist, then it means we haven't put any records
    # into this authority yet.

    if not table.exists():
        report.trace("Import: creating table for authority " + auth_name)
        table.create()

    return table

def get_pl_pubkey(key_id):
    keys = shell.GetKeys(pl_auth, [key_id])
    if keys:
        key_str = keys[0]['key']

        if "ssh-dss" in key_str:
            print "XXX: DSA key encountered, ignoring"
            return None

        # generate temporary files to hold the keys
        (ssh_f, ssh_fn) = tempfile.mkstemp()
        ssl_fn = tempfile.mktemp()

        os.write(ssh_f, key_str)
        os.close(ssh_f)

        if not os.path.exists(keyconvert_fn):
            report.trace("  keyconvert utility " + str(keyconvert_fn) + " does not exist");
            sys.exit(-1)

        cmd = keyconvert_fn + " " + ssh_fn + " " + ssl_fn
        print cmd
        os.system(cmd)

        # this check leaves the temporary file containing the public key so
        # that it can be expected to see why it failed.
        # TODO: for production, cleanup the temporary files
        if not os.path.exists(ssl_fn):
            report.trace("  failed to convert key from " + ssh_fn + " to " + ssl_fn)
            return None

        k = Keypair()
        try:
            k.load_pubkey_from_file(ssl_fn)
        except:
            print "XXX: Error while converting key: ", key_str
            k = None

        # remove the temporary files
        os.remove(ssh_fn)
        os.remove(ssl_fn)

        return k
    else:
        return None

def person_to_hrn(parent_hrn, person):
    # the old way - Lastname_Firstname
    #personname = person['last_name'] + "_" + person['first_name']

    # the new way - use email address up to the "@"
    personname = person['email'].split("@")[0]

    personname = cleanup_string(personname)

    hrn = parent_hrn + "." + personname
    return hrn

def import_person(parent_hrn, person):
    AuthHierarchy = Hierarchy()
    hrn = person_to_hrn(parent_hrn, person)

    # ASN.1 will have problems with hrn's longer than 64 characters
    if len(hrn) > 64:
        hrn = hrn[:64]

    report.trace("Import: importing person " + hrn)

    table = get_auth_table(parent_hrn)

    person_record = table.resolve("user", hrn)
    if not person_record:
        key_ids = []
        if 'key_ids' in person:
            key_ids = person["key_ids"]

        if key_ids:
            # get the user's private key from the SSH keys they have uploaded
            # to planetlab
            pkey = get_pl_pubkey(key_ids[0])
        else:
            # the user has no keys
            report.trace("   person " + hrn + " does not have a PL public key")
            pkey = None

        # if a key is unavailable, then we still need to put something in the
        # user's GID. So make one up.
        if not pkey:
            pkey = Keypair(create=True)

        person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
        report.trace("  inserting user record for " + hrn)
        table.insert(person_record)
    else:
        key_ids = person["key_ids"]

    if key_ids:
        pkey = get_pl_pubkey(key_ids[0])
        person_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        person_record = GeniRecord(name=hrn, gid=person_gid, type="user", pointer=person['person_id'])
        report.trace("  updating user record for " + hrn)
        table.update(person_record)

def import_slice(parent_hrn, slice):
    AuthHierarchy = Hierarchy()
    slicename = slice['name'].split("_",1)[-1]
    slicename = cleanup_string(slicename)

    if not slicename:
        report.error("Import_Slice: failed to parse slice name " + slice['name'])
        return

    hrn = parent_hrn + "." + slicename
    report.trace("Import: importing slice " + hrn)

    table = get_auth_table(parent_hrn)

    slice_record = table.resolve("slice", hrn)
    if not slice_record:
        pkey = Keypair(create=True)
        slice_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        slice_record = GeniRecord(name=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'])
        report.trace("  inserting slice record for " + hrn)
        table.insert(slice_record)

def import_node(parent_hrn, node):
    AuthHierarchy = Hierarchy()
    nodename = node['hostname'].split(".")[0]
    nodename = cleanup_string(nodename)

    if not nodename:
        report.error("Import_node: failed to parse node name " + node['hostname'])
        return

    hrn = parent_hrn + "." + nodename

    # ASN.1 will have problems with hrn's longer than 64 characters
    if len(hrn) > 64:
        hrn = hrn[:64]

    report.trace("Import: importing node " + hrn)

    table = get_auth_table(parent_hrn)

    node_record = table.resolve("node", hrn)
    if not node_record:
        pkey = Keypair(create=True)
        node_gid = AuthHierarchy.create_gid(hrn, create_uuid(), pkey)
        node_record = GeniRecord(name=hrn, gid=node_gid, type="node", pointer=node['node_id'])
        report.trace("  inserting node record for " + hrn)
        table.insert(node_record)

def import_site(parent_hrn, site):
    AuthHierarchy = Hierarchy()
    sitename = site['login_base']
    sitename = cleanup_string(sitename)

    hrn = parent_hrn + "." + sitename

    report.trace("Import_Site: importing site " + hrn)

    # create the authority
    if not AuthHierarchy.auth_exists(hrn):
        AuthHierarchy.create_auth(hrn)

    auth_info = AuthHierarchy.get_auth_info(hrn)

    table = get_auth_table(parent_hrn)

    auth_record = table.resolve("authority", hrn)
    if not auth_record:
        auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id'])
        report.trace("  inserting authority record for " + hrn)
        table.insert(auth_record)

    if 'person_ids' in site:
        for person_id in site['person_ids']:
            persons = shell.GetPersons(pl_auth, [person_id])
            if persons:
                try:
                    import_person(hrn, persons[0])
                except:
                    report.trace("Failed to import: %s" % persons[0])
    if 'slice_ids' in site:
        for slice_id in site['slice_ids']:
            slices = shell.GetSlices(pl_auth, [slice_id])
            if slices:
                try:
                    import_slice(hrn, slices[0])
                except:
                    report.trace("Failed to import: %s" % slices[0])
    if 'node_ids' in site:
        for node_id in site['node_ids']:
            nodes = shell.GetNodes(pl_auth, [node_id])
            if nodes:
                try:
                    import_node(hrn, nodes[0])
                except:
                    report.trace("Failed to import: %s" % nodes[0])

def create_top_level_auth_records(hrn):
    parent_hrn = get_authority(hrn)
    print hrn, ":", parent_hrn
    if not parent_hrn:
        parent_hrn = hrn
    auth_info = AuthHierarchy.get_auth_info(parent_hrn)
    table = get_auth_table(parent_hrn)

    auth_record = table.resolve("authority", hrn)
    if not auth_record:
        auth_record = GeniRecord(name=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1)
        report.trace("  inserting authority record for " + hrn)
        table.insert(auth_record)

def main():
    global AuthHierarchy
    global TrustedRoots

    process_options()

    print "Base Directory: ", config.GENI_BASE_DIR

    AuthHierarchy = Hierarchy()
    TrustedRoots = TrustedRootList()

    print "Import: creating top level authorities"

    if not AuthHierarchy.auth_exists(root_auth):
        AuthHierarchy.create_auth(root_auth)

    create_top_level_auth_records(root_auth)
    if level1_auth:
        if not AuthHierarchy.auth_exists(level1_auth):
            AuthHierarchy.create_auth(level1_auth)
        create_top_level_auth_records(level1_auth)
        import_auth = level1_auth
    else:
        import_auth = root_auth

    print "Import: adding", root_auth, "to trusted list"
    root = AuthHierarchy.get_auth_info(root_auth)
    TrustedRoots.add_gid(root.get_gid_object())

    connect_shell()

    sites = shell.GetSites(pl_auth)
    for site in sites:
        import_site(import_auth, site)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Create a virtualenv.

Create a virtualenv using the current working direction as a name, which is
typically the project that you're isolating for development.

Use *vinit* in the following way.

  $ git clone https://github.com/sholsapp/powertools
  $ cd powertools
  $ vinit -r tox -r requests
  $ source activate
  $ python setup.py develop

"""

import argparse
import logging
import os
import shutil
import subprocess
import sys

logging.basicConfig(level=logging.INFO)

log = logging.getLogger(__name__)

default_virtualenv = os.path.join(
  os.path.expanduser('~'),
  '.virtualenvs',
  os.path.basename(os.getcwd())
)

def main():

  parser = argparse.ArgumentParser(description=__doc__,
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-d', '--dest-directory', default=default_virtualenv,
                      help='Where to create the virtualenv instead of %(default)s.')
  parser.add_argument('-n', '--new', action='store_true',
                      help='Create a new virtualenv if it already exists.')
  parser.add_argument('-r', '--requirement', action='append', default=[],
                      help='Pip-style requirements to bootstrap the virtualenv with.')
  args = parser.parse_args()

  if not os.path.exists(args.dest_directory) or args.new:
    if os.path.exists(args.dest_directory):
      shutil.rmtree(args.dest_directory)
    subprocess.check_call(['virtualenv', args.dest_directory])
    subprocess.check_call([
      os.path.join(args.dest_directory, 'bin', 'pip'),
      'install', 'pip', '--upgrade'])

  for requirement in args.requirement:
    subprocess.check_call([
      os.path.join(args.dest_directory, 'bin', 'pip'),
      'install', requirement])

  if not os.path.exists('activate'):
    subprocess.check_call(['ln', '-s', os.path.join(args.dest_directory, 'bin', 'activate'), 'activate'])

  print u"Congratulations! \U0001F37A"
<commit_msg>Add --system-site-packages option to vinit.<commit_after>#!/usr/bin/env python

"""Create a virtualenv.

Create a virtualenv using the current working direction as a name, which is
typically the project that you're isolating for development.

Use *vinit* in the following way.

  $ git clone https://github.com/sholsapp/powertools
  $ cd powertools
  $ vinit -r tox -r requests
  $ source activate
  $ python setup.py develop

"""

import argparse
import logging
import os
import shutil
import subprocess
import sys

logging.basicConfig(level=logging.INFO)

log = logging.getLogger(__name__)

default_virtualenv = os.path.join(
  os.path.expanduser('~'),
  '.virtualenvs',
  os.path.basename(os.getcwd())
)

def main():

  parser = argparse.ArgumentParser(description=__doc__,
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-d', '--dest-directory', default=default_virtualenv,
                      help='Where to create the virtualenv instead of %(default)s.')
  parser.add_argument('-n', '--new', action='store_true',
                      help='Create a new virtualenv if it already exists.')
  parser.add_argument('-r', '--requirement', action='append', default=[],
                      help='Pip-style requirements to bootstrap the virtualenv with.')
  parser.add_argument('--system-site-packages', action='store_true',
                      help='Wrapper over virtualenv --system-site-packages option.')
  args = parser.parse_args()

  if not os.path.exists(args.dest_directory) or args.new:
    if os.path.exists(args.dest_directory):
      shutil.rmtree(args.dest_directory)
    virtualenv_cmd = ['virtualenv']
    if args.system_site_packages:
      virtualenv_cmd.append('--system-site-packages')
    virtualenv_cmd.append(args.dest_directory)
    subprocess.check_call(virtualenv_cmd)
    subprocess.check_call([
      os.path.join(args.dest_directory, 'bin', 'pip'),
      'install', 'pip', '--upgrade'])

  for requirement in args.requirement:
    subprocess.check_call([
      os.path.join(args.dest_directory, 'bin', 'pip'),
      'install', requirement])

  if not os.path.exists('activate'):
    subprocess.check_call(['ln', '-s', os.path.join(args.dest_directory, 'bin', 'activate'), 'activate'])

  print u"Congratulations! \U0001F37A"
<|endoftext|>


<commit_before>from pprof.settings import config

def lt_clang(cflags, ldflags, func=None):
    """Return a clang that hides :cflags: and :ldflags: from reordering of
    libtool.

    This will generate a wrapper script in :p:'s builddir and return a path
    to it.

    :cflags: the cflags libtool is not allowed to see.
    :ldflags: the ldflags libtool is not allowed to see.
    :returns: path to the new clang.

    """
    from plumbum import local

    print_libtool_sucks_wrapper("clang", cflags, ldflags, clang, func)
    return local["./clang"]

def lt_clang_cxx(cflags, ldflags, func=None):
    """Return a clang that hides :cflags: and :ldflags: from reordering of
    libtool.

    This will generate a wrapper script in :p:'s builddir and return a path
    to it.

    :cflags: the cflags libtool is not allowed to see.
    :ldflags: the ldflags libtool is not allowed to see.
    :returns: path to the new clang.

    """
    from plumbum import local

    print_libtool_sucks_wrapper("clang++", cflags, ldflags, clang_cxx, func)
    return local["./clang++"]

def print_libtool_sucks_wrapper(filepath, cflags, ldflags, compiler, func):
    """Print a libtool wrapper that hides :flags_to_hide: from libtool.

    :filepath:
        Where should the new compiler be?
    :flags_to_hide:
        List of flags that should be hidden from libtool
    :compiler:
        The compiler we should actually call
    """
    from plumbum.cmd import chmod
    from cloud.serialization import cloudpickle as cp
    from pprof.project import PROJECT_BLOB_F_EXT
    from os.path import abspath

    blob_f = abspath(filepath + PROJECT_BLOB_F_EXT)
    if func is not None:
        with open(blob_f, 'wb') as b:
            b.write(cp.dumps(func))

    with open(filepath, 'w') as wrapper:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import ProcessExecutionError, local, FG
from pprof.experiment import to_utf8
from os import path
import pickle

from pprof.settings import config
config["db_host"] = "{db_host}"
config["db_port"] = "{db_port}"
config["db_name"] = "{db_name}"
config["db_user"] = "{db_user}"
config["db_pass"] = "{db_pass}"

cc=local[\"{CC}\"]
cflags={CFLAGS}
ldflags={LDFLAGS}

from sys import argv
import os
import sys

def call_original_compiler(input_files, cc, cflags, ldflags, flags):
    final_command = None
    retcode=0
    try:
        if len(input_files) > 0:
            if "-c" in flags:
                final_command = cc["-Qunused-arguments", cflags, flags]
            else:
                final_command = cc["-Qunused-arguments", cflags, flags, ldflags]
        else:
            final_command = cc["-Qunused-arguments", flags]

        retcode, stdout, stderr = final_command.run()
        if len(stdout) > 0:
            print stdout
        if len(stderr) > 0:
            print stderr
    except ProcessExecutionError as e:
        sys.stderr.write(to_utf8(str(e.stderr)))
        sys.stderr.flush()
        sys.exit(e.retcode)
    return (retcode, final_command)

input_files = [ x for x in argv[1:] if not '-' is x[0] ]
flags = argv[1:]
f = None

with local.env(PPROF_DB_HOST="{db_host}",
           PPROF_DB_PORT="{db_port}",
           PPROF_DB_NAME="{db_name}",
           PPROF_DB_USER="{db_user}",
           PPROF_DB_PASS="{db_pass}"):
    retcode, final_cc = call_original_compiler(input_files, cc, cflags,
                                               ldflags, flags)
    """ FIXME: This is just a quick workaround. """
    if "conftest.c" not in input_files:
        with local.env(PPROF_CMD=str(final_cc)):
            if path.exists("{blobf}"):
                with open("{blobf}", "rb") as p:
                    f = pickle.load(p)

            if f is not None:
                if not sys.stdin.isatty():
                    f(final_cc, has_stdin = True)
                else:
                    f(final_cc)
    sys.exit(retcode)
'''.format(CC=str(compiler()), CFLAGS=cflags, LDFLAGS=ldflags,
           blobf=blob_f, db_host=config["db_host"],
           db_name=config["db_name"], db_user=config["db_user"],
           db_pass=config["db_pass"], db_port=config["db_port"])
        wrapper.write(lines)
        chmod("+x", filepath)

def llvm():
    from os import path
    return path.join(config["llvmdir"], "bin")

def llvm_libs():
    from os import path
    return path.join(config["llvmdir"], "lib")

def clang_cxx():
    from os import path
    from plumbum import local
    return local[path.join(llvm(), "clang++")]

def clang():
    from os import path
    from plumbum import local
    return local[path.join(llvm(), "clang")]
<commit_msg>compiler: fallback to default cc on failure<commit_after>from pprof.settings import config

def lt_clang(cflags, ldflags, func=None):
    """Return a clang that hides :cflags: and :ldflags: from reordering of
    libtool.

    This will generate a wrapper script in :p:'s builddir and return a path
    to it.

    :cflags: the cflags libtool is not allowed to see.
    :ldflags: the ldflags libtool is not allowed to see.
    :returns: path to the new clang.

    """
    from plumbum import local

    print_libtool_sucks_wrapper("clang", cflags, ldflags, clang, func)
    return local["./clang"]

def lt_clang_cxx(cflags, ldflags, func=None):
    """Return a clang that hides :cflags: and :ldflags: from reordering of
    libtool.

    This will generate a wrapper script in :p:'s builddir and return a path
    to it.

    :cflags: the cflags libtool is not allowed to see.
    :ldflags: the ldflags libtool is not allowed to see.
    :returns: path to the new clang.

    """
    from plumbum import local

    print_libtool_sucks_wrapper("clang++", cflags, ldflags, clang_cxx, func)
    return local["./clang++"]

def print_libtool_sucks_wrapper(filepath, cflags, ldflags, compiler, func):
    """Print a libtool wrapper that hides :flags_to_hide: from libtool.

    :filepath:
        Where should the new compiler be?
    :flags_to_hide:
        List of flags that should be hidden from libtool
    :compiler:
        The compiler we should actually call
    """
    from plumbum.cmd import chmod
    from cloud.serialization import cloudpickle as cp
    from pprof.project import PROJECT_BLOB_F_EXT
    from os.path import abspath

    blob_f = abspath(filepath + PROJECT_BLOB_F_EXT)
    if func is not None:
        with open(blob_f, 'wb') as b:
            b.write(cp.dumps(func))

    with open(filepath, 'w') as wrapper:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import ProcessExecutionError, local, FG
from pprof.experiment import to_utf8
from os import path
import pickle

from pprof.settings import config
config["db_host"] = "{db_host}"
config["db_port"] = "{db_port}"
config["db_name"] = "{db_name}"
config["db_user"] = "{db_user}"
config["db_pass"] = "{db_pass}"

cc=local[\"{CC}\"]
cflags={CFLAGS}
ldflags={LDFLAGS}

from sys import argv
import os
import sys

def call_original_compiler(input_files, cc, cflags, ldflags, flags):
    final_command = None
    retcode=0
    try:
        if len(input_files) > 0:
            if "-c" in flags:
                final_command = cc["-Qunused-arguments", cflags, flags]
            else:
                final_command = cc["-Qunused-arguments", cflags, flags,
                                   ldflags]
        else:
            final_command = cc["-Qunused-arguments", flags]

        retcode, stdout, stderr = final_command.run()
        if len(stdout) > 0:
            print stdout
        if len(stderr) > 0:
            print stderr
    except ProcessExecutionError:
        cc(flags)
    return (retcode, final_command)

input_files = [ x for x in argv[1:] if not '-' is x[0] ]
flags = argv[1:]
f = None

with local.env(PPROF_DB_HOST="{db_host}",
           PPROF_DB_PORT="{db_port}",
           PPROF_DB_NAME="{db_name}",
           PPROF_DB_USER="{db_user}",
           PPROF_DB_PASS="{db_pass}"):
    retcode, final_cc = call_original_compiler(input_files, cc, cflags,
                                               ldflags, flags)
    """ FIXME: This is just a quick workaround. """
    if "conftest.c" not in input_files:
        with local.env(PPROF_CMD=str(final_cc)):
            if path.exists("{blobf}"):
                with open("{blobf}", "rb") as p:
                    f = pickle.load(p)

            if f is not None:
                if not sys.stdin.isatty():
                    f(final_cc, has_stdin = True)
                else:
                    f(final_cc)
    sys.exit(retcode)
'''.format(CC=str(compiler()), CFLAGS=cflags, LDFLAGS=ldflags,
           blobf=blob_f, db_host=config["db_host"],
           db_name=config["db_name"], db_user=config["db_user"],
           db_pass=config["db_pass"], db_port=config["db_port"])
        wrapper.write(lines)
        chmod("+x", filepath)

def llvm():
    from os import path
    return path.join(config["llvmdir"], "bin")

def llvm_libs():
    from os import path
    return path.join(config["llvmdir"], "lib")

def clang_cxx():
    from os import path
    from plumbum import local
    return local[path.join(llvm(), "clang++")]

def clang():
    from os import path
    from plumbum import local
    return local[path.join(llvm(), "clang")]
<|endoftext|>


<commit_before>
from geopy.distance import vincenty
import data

class City(object):
    '''
    Representation of a city data structure.
    '''
    def __init__(self, datadict):
        self.datadict = datadict
        for i,_ in data.HEADER:
            setattr(self, i, self.datadict[i])

    def display(self):
        print "******Cityid:", self.geonameid, "****"
        for attribute, _ in data.HEADER:
            value = getattr(self, attribute)
            print '    ', attribute, ':', value

    def __sub__(self, cityobj):
        '''
        Overload minus sign used as a distance method name
        '''
        if type(cityobj) == City:
            c1 = (float(self.longitude), float(self.latitude))
            c2 = (float(cityobj.longitude), float(cityobj.latitude))
            #from geopy.distance import great_circle
            #return great_circle(c1, c2)
            return vincenty(c1, c2).miles

if __name__ == '__main__':
    citydata = data.CityData()
    citylist = []

    for idd, cityobj in citydata.result.items():
        citylist.append(cityobj)
    c1 = citylist[0]
    c1.display()

    c2 = citylist[1]
    c2.display()

    # computing distance
    d1 = c1-c2
    d2 = c2-c1
    print 'distance c1-c2', d1
    print 'distance c2-c1', d2
    d3 = c1-c1
    print 'distance c1-c1', d3

    print d1 > d3, "should be true"
    print d1 == d2, "should be true"
<commit_msg>Fixed problem on geo distance method call<commit_after>
from geopy.distance import vincenty
import data

class City(object):
    '''
    Representation of a city data structure.
    '''
    def __init__(self, datadict):
        self.datadict = datadict
        for i,_ in data.HEADER:
            setattr(self, i, self.datadict[i])

    def display(self):
        print "******Cityid:", self.geonameid, "****"
        for attribute, _ in data.HEADER:
            value = getattr(self, attribute)
            print '    ', attribute, ':', value

    def __sub__(self, cityobj):
        '''
        Overload minus sign used as a distance method name.  Here the distance
        function 'vincenty' is used to calculate the distance between pairs of
        geo (longitude, latitude)
        '''
        if type(cityobj) == City:
            c1 = (float(self.latitude), float(self.longitude))
            c2 = (float(cityobj.latitude), float(cityobj.longitude))
            #from geopy.distance import great_circle
            #return great_circle(c1, c2)
            return vincenty(c1, c2).miles

if __name__ == '__main__':
    citydata = data.CityData()
    citylist = []

    for idd, cityobj in citydata.result.items():
        citylist.append(cityobj)
    c1 = citylist[0]
    c1.display()

    c2 = citylist[1]
    c2.display()

    # computing distance
    d1 = c1-c2
    d2 = c2-c1
    print 'distance c1-c2', d1
    print 'distance c2-c1', d2
    d3 = c1-c1
    print 'distance c1-c1', d3

    print d1 > d3, "should be true"
    print d1 == d2, "should be true"
<|endoftext|>


<commit_before>"""
Created on 27 Sep 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import re
import json

from collections import OrderedDict
from copy import deepcopy

from scs_core.data.json import JSONable

# --------------------------------------------------------------------------------------------------------------------

class PathDict(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jstr(cls, jstr):
        try:
            jdict = json.loads(jstr, object_pairs_hook=OrderedDict)
        except ValueError:
            return None

        return PathDict(jdict)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, dictionary=None):
        """
        Constructor
        """
        self.__dictionary = OrderedDict() if dictionary is None else dictionary

    def __len__(self):
        return len(self.__dictionary)

    # -----------------------------------------------------------------------------------------------------------------
    # source...

    # Tests whether a leaf-node path is present...

    def has_path(self, path):
        return path in self.paths()

    # Tests whether a leaf-node or internal path is present...

    def has_sub_path(self, sub_path=None):
        if sub_path is None:
            return True

        try:
            self.node(sub_path)
            return True

        except KeyError:
            return False

    # Returns all the leaf-node paths...

    def paths(self, sub_path=None):
        node = self if sub_path is None else PathDict(self.node(sub_path))

        return node.__paths(node.__dictionary, sub_path)

    # Returns a leaf-node or internal node...

    def node(self, sub_path=None):
        if sub_path is None:
            return self.__dictionary

        return self.__node(self.__dictionary, re.split(r"[.:]", sub_path))

    # ----------------------------------------------------------------------------------------------------------------
    # target...

    def copy(self, other, sub_path=None):
        if sub_path is None:
            self.__dictionary = deepcopy(other.__dictionary)
            return

        self.__append(self.__dictionary, re.split(r"[.:]", sub_path), other.node(sub_path))

    def append(self, sub_path, value):
        nodes = re.findall('([^.:]+)([.:]*)', sub_path)

        self.__append(self.__dictionary, nodes, value)

    # ----------------------------------------------------------------------------------------------------------------

    def __paths(self, container, prefix=None):
        # dict...
        if isinstance(container, dict):
            separator = '.'
            keys = container.keys()

        # list...
        elif isinstance(container, list):
            separator = ':'
            keys = range(len(container))

        # scalar...
        else:
            return [prefix]

        prefix = prefix + separator if prefix else ''

        paths = []

        for key in keys:
            path = prefix + str(key)
            value = container[key]

            # dict...
            if isinstance(value, dict):
                paths.extend(self.__paths(value, path))

            # list...
            elif isinstance(value, list):
                for i in range(len(value)):
                    if isinstance(value[i], (list, dict)):
                        paths.extend(self.__paths(value[i], path + ':' + str(i)))
                    else:
                        paths.append(path + ':' + str(i))

            # scalar...
            else:
                paths.append(path)

        return paths

    def __node(self, container, nodes):
        # key...
        if isinstance(container, list):
            try:
                key = int(nodes[0])

            except ValueError:
                raise KeyError(nodes[0])            # a non-integer key on an array is a KeyError

        else:
            key = nodes[0]

        # value...
        value = container[key]

        # leaf...
        if len(nodes) == 1:
            return value

        # deep...
        return self.__node(value, nodes[1:])

    def __append(self, container, nodes, value):
        # key...
        key = nodes[0][0]
        separator = nodes[0][1]

        # dict...
        if isinstance(container, dict):
            if key not in container:
                container[key] = [] if separator == ':' else OrderedDict()

        # list...
        elif isinstance(container, list):
            try:
                key = int(key)
            except ValueError:
                raise KeyError(key)

            if key >= len(container):
                container.append([] if separator == ':' else OrderedDict())
                key = len(container) - 1

        # leaf...
        if len(nodes) == 1:
            container[key] = deepcopy(value)
            return

        # deep...
        self.__append(container[key], nodes[1:], value)

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def dictionary(self):
        return self.node()

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        return self.__dictionary

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "PathDict:{dictionary:%s}" % (self.node())
<commit_msg>Improved array index handling in PathDict.<commit_after>"""
Created on 27 Sep 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import re
import json

from collections import OrderedDict
from copy import deepcopy

from scs_core.data.json import JSONable

# --------------------------------------------------------------------------------------------------------------------

class PathDict(JSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct_from_jstr(cls, jstr):
        try:
            jdict = json.loads(jstr, object_pairs_hook=OrderedDict)
        except ValueError:
            return None

        return PathDict(jdict)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, dictionary=None):
        """
        Constructor
        """
        self.__dictionary = OrderedDict() if dictionary is None else dictionary

    def __len__(self):
        return len(self.__dictionary)

    # -----------------------------------------------------------------------------------------------------------------
    # source...

    # Tests whether a leaf-node path is present...

    def has_path(self, path):
        return path in self.paths()

    # Tests whether a leaf-node or internal path is present...

    def has_sub_path(self, sub_path=None):
        if sub_path is None:
            return True

        try:
            self.node(sub_path)
            return True

        except KeyError:
            return False

    # Returns all the leaf-node paths...

    def paths(self, sub_path=None):
        node = self if sub_path is None else PathDict(self.node(sub_path))

        return node.__paths(node.__dictionary, sub_path)

    # Returns a leaf-node or internal node...

    def node(self, sub_path=None):
        if sub_path is None:
            return self.__dictionary

        return self.__node(self.__dictionary, re.split(r"[.:]", sub_path))

    # ----------------------------------------------------------------------------------------------------------------
    # target...

    def copy(self, other, sub_path=None):
        if sub_path is None:
            self.__dictionary = deepcopy(other.__dictionary)
            return

        self.__append(self.__dictionary, re.split(r"[.:]", sub_path), other.node(sub_path))

    def append(self, sub_path, value):
        nodes = re.findall('([^.:]+)([.:]*)', sub_path)

        self.__append(self.__dictionary, nodes, value)

    # ----------------------------------------------------------------------------------------------------------------

    def __paths(self, container, prefix=None):
        # dict...
        if isinstance(container, dict):
            separator = '.'
            keys = container.keys()

        # list...
        elif isinstance(container, list):
            separator = ':'
            keys = range(len(container))

        # scalar...
        else:
            return [prefix]

        prefix = prefix + separator if prefix else ''

        paths = []

        for key in keys:
            path = prefix + str(key)
            value = container[key]

            # dict...
            if isinstance(value, dict):
                paths.extend(self.__paths(value, path))

            # list...
            elif isinstance(value, list):
                for i in range(len(value)):
                    if isinstance(value[i], (list, dict)):
                        paths.extend(self.__paths(value[i], path + ':' + str(i)))
                    else:
                        paths.append(path + ':' + str(i))

            # scalar...
            else:
                paths.append(path)

        return paths

    def __node(self, container, nodes):
        # key...
        if isinstance(container, list):
            try:
                key = int(nodes[0])

            except ValueError:
                raise KeyError(nodes[0])            # a non-integer key on an array is a KeyError

        else:
            key = nodes[0]

        # value...
        value = container[key]

        # leaf...
        if len(nodes) == 1:
            return value

        # deep...
        return self.__node(value, nodes[1:])

    def __append(self, container, nodes, value):
        # key...
        key = nodes[0][0]
        separator = nodes[0][1]

        # dict...
        if isinstance(container, dict):
            if key not in container:
                container[key] = [] if separator == ':' else OrderedDict()

        # list...
        elif isinstance(container, list):
            try:
                key = int(key)
            except ValueError:
                raise KeyError(key)

            while key >= len(container):
                container.append(None)

            container[key] = [] if separator == ':' else OrderedDict()

        # leaf...
        if len(nodes) == 1:
            container[key] = deepcopy(value)
            return

        # deep...
        self.__append(container[key], nodes[1:], value)

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def dictionary(self):
        return self.node()

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        return self.__dictionary

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "PathDict:{dictionary:%s}" % (self.node())
<|endoftext|>


<commit_before>"""
    ulid/api
    ~~~~~~~~

    Defines the public API of the `ulid` package.
"""
import os
import time
import uuid

from . import base32, hints, ulid

__all__ = ['new', 'from_bytes', 'from_int', 'from_str', 'from_uuid', 'from_timestamp', 'from_randomness']

def new() -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance.

    The timestamp is created from :func:`~time.time`.
    The randomness is created from :func:`~os.urandom`.

    :return: ULID from current timestamp
    :rtype: :class:`~ulid.ulid.ULID`
    """
    timestamp = int(time.time() * 1000).to_bytes(6, byteorder='big')
    randomness = os.urandom(10)
    return ulid.ULID(timestamp + randomness)

def from_bytes(value: hints.Buffer) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~bytes`,
    :class:`~bytearray`, or :class:`~memoryview` value.

    :param value: 16 bytes
    :type value: :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID from buffer value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not 16 bytes
    """
    length = len(value)
    if length != 16:
        raise ValueError('Expects bytes to be 128 bits; got {} bytes'.format(length))

    return ulid.ULID(value)

def from_int(value: int) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~int` value.

    :param value: 128 bit integer
    :type value: :class:`~int`
    :return: ULID from integer value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not a 128 bit integer
    """
    length = (value.bit_length() + 7) // 8
    if length != 16:
        raise ValueError('Expects integer to be 128 bits; got {} bytes'.format(length))

    return ulid.ULID(value.to_bytes(16, byteorder='big'))

def from_str(value: str) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~str` value.

    :param value: Base32 encoded string
    :type value: :class:`~str`
    :return: ULID from string value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not 26 characters or malformed
    """
    return ulid.ULID(base32.decode_ulid(value))

def from_uuid(value: uuid.UUID) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~uuid.UUID` value.

    :param value: UUIDv4 value
    :type value: :class:`~uuid.UUID`
    :return: ULID from UUID value
    :rtype: :class:`~ulid.ulid.ULID`
    """
    return ulid.ULID(value.bytes)

def from_timestamp(timestamp: hints.TimestampPrimitive) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance using the given :class:`~int`, :class:`~float`,
    :class:`~str`, :class:`~bytes`, :class:`~bytearray`, or :class`~memoryview` value that
    represents Unix timestamp in seconds.

    :param timestamp: Unix timestamp in seconds
    :type timestamp: :class:`~int`, :class:`~float`, :class:`~str`,
        :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID using given timestamp and new randomness
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is an unsupported type
    :raises ValueError: when the value is a string and cannot be Base32 decoded
    :raises ValueError: when the value is or was converted to something 48 bits
    """
    if isinstance(timestamp, (int, float)):
        timestamp = int(timestamp * 1000.0).to_bytes(6, byteorder='big')
    elif isinstance(timestamp, str):
        timestamp = base32.decode_timestamp(timestamp)

    if not isinstance(timestamp, (bytes, bytearray, memoryview)):
        raise ValueError('Expected int, float, str, bytes, '
                         'bytearray, or memoryview; got {}'.format(type(timestamp).__name__))

    length = len(timestamp)
    if length != 6:
        raise ValueError('Expects timestamp to be 48 bits; got {} bytes'.format(length))

    randomness = os.urandom(10)
    return ulid.ULID(timestamp + randomness)

def from_randomness(randomness: hints.Buffer) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance using the given :class:`~int`, :class:`~float`,
    :class:`~str`, :class:`~bytes`, :class:`~bytearray`, or :class`~memoryview` value that
    represents cryptographically secure random values.

    :param randomness: Random bytes
    :type randomness: :class:`~int`, :class:`~float`, :class:`~str`,
        :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID using new timestamp and given randomness
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is an unsupported type
    :raises ValueError: when the value is a string and cannot be Base32 decoded
    :raises ValueError: when the value is or was converted to something 80 bits
    """
    if isinstance(randomness, (int, float)):
        randomness = int(randomness).to_bytes(10, byteorder='big')
    elif isinstance(randomness, str):
        randomness = base32.decode_randomness(randomness)

    if not isinstance(randomness, (bytes, bytearray, memoryview)):
        raise ValueError('Expected int, float, str, bytes, '
                         'bytearray, or memoryview; got {}'.format(type(randomness).__name__))

    length = len(randomness)
    if length != 10:
        raise ValueError('Expects randomness to be 80 bits; got {} bytes'.format(length))

    timestamp = int(time.time() * 1000).to_bytes(6, byteorder='big')
    return ulid.ULID(timestamp + randomness)
<commit_msg>Fix incorrect from_randomness parameter type hint.<commit_after>"""
    ulid/api
    ~~~~~~~~

    Defines the public API of the `ulid` package.
"""
import os
import time
import uuid

from . import base32, hints, ulid

__all__ = ['new', 'from_bytes', 'from_int', 'from_str', 'from_uuid', 'from_timestamp', 'from_randomness']

def new() -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance.

    The timestamp is created from :func:`~time.time`.
    The randomness is created from :func:`~os.urandom`.

    :return: ULID from current timestamp
    :rtype: :class:`~ulid.ulid.ULID`
    """
    timestamp = int(time.time() * 1000).to_bytes(6, byteorder='big')
    randomness = os.urandom(10)
    return ulid.ULID(timestamp + randomness)

def from_bytes(value: hints.Buffer) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~bytes`,
    :class:`~bytearray`, or :class:`~memoryview` value.

    :param value: 16 bytes
    :type value: :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID from buffer value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not 16 bytes
    """
    length = len(value)
    if length != 16:
        raise ValueError('Expects bytes to be 128 bits; got {} bytes'.format(length))

    return ulid.ULID(value)

def from_int(value: int) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~int` value.

    :param value: 128 bit integer
    :type value: :class:`~int`
    :return: ULID from integer value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not a 128 bit integer
    """
    length = (value.bit_length() + 7) // 8
    if length != 16:
        raise ValueError('Expects integer to be 128 bits; got {} bytes'.format(length))

    return ulid.ULID(value.to_bytes(16, byteorder='big'))

def from_str(value: str) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~str` value.

    :param value: Base32 encoded string
    :type value: :class:`~str`
    :return: ULID from string value
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is not 26 characters or malformed
    """
    return ulid.ULID(base32.decode_ulid(value))

def from_uuid(value: uuid.UUID) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance from the given :class:`~uuid.UUID` value.

    :param value: UUIDv4 value
    :type value: :class:`~uuid.UUID`
    :return: ULID from UUID value
    :rtype: :class:`~ulid.ulid.ULID`
    """
    return ulid.ULID(value.bytes)

def from_timestamp(timestamp: hints.TimestampPrimitive) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance using the given :class:`~int`, :class:`~float`,
    :class:`~str`, :class:`~bytes`, :class:`~bytearray`, or :class`~memoryview` value that
    represents Unix timestamp in seconds.

    :param timestamp: Unix timestamp in seconds
    :type timestamp: :class:`~int`, :class:`~float`, :class:`~str`,
        :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID using given timestamp and new randomness
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is an unsupported type
    :raises ValueError: when the value is a string and cannot be Base32 decoded
    :raises ValueError: when the value is or was converted to something 48 bits
    """
    if isinstance(timestamp, (int, float)):
        timestamp = int(timestamp * 1000.0).to_bytes(6, byteorder='big')
    elif isinstance(timestamp, str):
        timestamp = base32.decode_timestamp(timestamp)

    if not isinstance(timestamp, (bytes, bytearray, memoryview)):
        raise ValueError('Expected int, float, str, bytes, '
                         'bytearray, or memoryview; got {}'.format(type(timestamp).__name__))

    length = len(timestamp)
    if length != 6:
        raise ValueError('Expects timestamp to be 48 bits; got {} bytes'.format(length))

    randomness = os.urandom(10)
    return ulid.ULID(timestamp + randomness)

def from_randomness(randomness: hints.RandomnessPrimitive) -> ulid.ULID:
    """
    Create a new :class:`~ulid.ulid.ULID` instance using the given :class:`~int`, :class:`~float`,
    :class:`~str`, :class:`~bytes`, :class:`~bytearray`, or :class`~memoryview` value that
    represents cryptographically secure random values.

    :param randomness: Random bytes
    :type randomness: :class:`~int`, :class:`~float`, :class:`~str`,
        :class:`~bytes`, :class:`~bytearray`, or :class:`~memoryview`
    :return: ULID using new timestamp and given randomness
    :rtype: :class:`~ulid.ulid.ULID`
    :raises ValueError: when the value is an unsupported type
    :raises ValueError: when the value is a string and cannot be Base32 decoded
    :raises ValueError: when the value is or was converted to something 80 bits
    """
    if isinstance(randomness, (int, float)):
        randomness = int(randomness).to_bytes(10, byteorder='big')
    elif isinstance(randomness, str):
        randomness = base32.decode_randomness(randomness)

    if not isinstance(randomness, (bytes, bytearray, memoryview)):
        raise ValueError('Expected int, float, str, bytes, '
                         'bytearray, or memoryview; got {}'.format(type(randomness).__name__))

    length = len(randomness)
    if length != 10:
        raise ValueError('Expects randomness to be 80 bits; got {} bytes'.format(length))

    timestamp = int(time.time() * 1000).to_bytes(6, byteorder='big')
    return ulid.ULID(timestamp + randomness)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 28 Feb 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The interface_power utility is used to simultaneously switch on and off the power to the following peripherals:

* GPS
* OPC
* NDIR
* modem (currently 2G only)
* integrated gas sensor interface (digital side only)

Note that not all interface board types are able to switch off all of these subsystems. Where switching is not possible,
no operation is performed, and no exception is raised.

LED control is direct - it bypasses the led_controller process, if running. This enables LED control in service stop
situations, where led_controller service may already have stopped.

The utility supports switch of any combination of peripherals, or all peripherals. Note that, in the case of 'all' the
modem is not included.

SYNOPSIS
interface_power.py { [-g ENABLE] [-p ENABLE] [-m ENABLE] [-n ENABLE] [-o ENABLE] [-l { R | A | G | 0 }] | ENABLE_ALL } \
[-v]

EXAMPLES
./interface_power.py 0

RESOURCES
https://github.com/south-coast-science/docs/wiki/Praxis-LED-colours
"""

import sys

from scs_core.sys.signalled_exit import SignalledExit

from scs_dev.cmd.cmd_power import CmdPower

from scs_dfe.gps.gps_conf import GPSConf
from scs_dfe.interface.interface_conf import InterfaceConf
from scs_dfe.particulate.opc_conf import OPCConf

from scs_host.bus.i2c import I2C
from scs_host.sys.host import Host

try:
    from scs_ndir.gas.ndir.ndir_conf import NDIRConf
except ImportError:
    from scs_core.gas.ndir.ndir_conf import NDIRConf

# --------------------------------------------------------------------------------------------------------------------

def power_gps(enable):
    if gps is None:
        return

    if enable:
        gps.power_on()
    else:
        gps.power_off()

def power_ndir(enable):
    if ndir is None:
        return

    if enable:
        ndir.power_on()
    else:
        ndir.power_off()

def power_opc(enable):
    if opc is None:
        return

    if enable:
        opc.power_on()
        opc.operations_on()
    else:
        opc.operations_off()
        opc.power_off()

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    I2C.open(Host.I2C_SENSORS)

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdPower()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("interface_power: %s" % cmd, file=sys.stderr)

    try:
        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # Interface...
        interface_conf = InterfaceConf.load(Host)

        if interface_conf is None:
            print("interface_power: InterfaceConf not available.", file=sys.stderr)
            exit(1)

        interface = interface_conf.interface()

        if cmd.verbose:
            print("interface_power: %s" % interface, file=sys.stderr)
            sys.stderr.flush()

        # GPS...
        gps_conf = GPSConf.load(Host)
        gps = None if gps_conf is None else gps_conf.gps(interface, Host)

        if cmd.verbose and gps:
            print("interface_power: %s" % gps, file=sys.stderr)

        # NDIR...
        ndir_conf = NDIRConf.load(Host)
        ndir = None if ndir_conf is None else ndir_conf.ndir(interface, Host)

        if cmd.verbose and ndir:
            print("interface_power: %s" % ndir, file=sys.stderr)

        # OPC...
        opc_conf = OPCConf.load(Host)
        opc = None if opc_conf is None else opc_conf.opc(interface, Host)

        if cmd.verbose and opc:
            print("interface_power: %s" % opc, file=sys.stderr)

        # LED...
        led = interface.led()

        if cmd.verbose and led:
            print("interface_power: %s" % led, file=sys.stderr)

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("interface_power", cmd.verbose)

        if cmd.all is not None:
            interface.power_gases(cmd.all)
            interface.power_modem(cmd.all)
            power_gps(cmd.all)
            power_ndir(cmd.all)
            power_opc(cmd.all)

        if cmd.gases is not None:
            interface.power_gases(cmd.gases)

        if cmd.gps is not None:
            power_gps(cmd.gps)

        if cmd.modem is not None:
            interface.power_modem(cmd.modem)

        if cmd.ndir is not None:
            power_ndir(cmd.ndir)

        if cmd.opc is not None:
            power_opc(cmd.opc)

        if cmd.led is not None:
            if led:
                led.colour = cmd.led

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("interface_power: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("interface_power: finishing", file=sys.stderr)

        I2C.close()
<commit_msg>Fixed a bug in interface_power.py<commit_after>#!/usr/bin/env python3

"""
Created on 28 Feb 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The interface_power utility is used to simultaneously switch on and off the power to the following peripherals:

* GPS
* OPC
* NDIR
* modem (currently 2G only)
* integrated gas sensor interface (digital side only)

Note that not all interface board types are able to switch off all of these subsystems. Where switching is not possible,
no operation is performed, and no exception is raised.

LED control is direct - it bypasses the led_controller process, if running. This enables LED control in service stop
situations, where led_controller service may already have stopped.

The utility supports switch of any combination of peripherals, or all peripherals. Note that, in the case of 'all' the
modem is not included.

SYNOPSIS
interface_power.py { [-g ENABLE] [-p ENABLE] [-m ENABLE] [-n ENABLE] [-o ENABLE] [-l { R | A | G | 0 }] | ENABLE_ALL } \
[-v]

EXAMPLES
./interface_power.py 0

RESOURCES
https://github.com/south-coast-science/docs/wiki/Praxis-LED-colours
"""

import sys

from scs_core.sys.signalled_exit import SignalledExit

from scs_dev.cmd.cmd_power import CmdPower

from scs_dfe.gps.gps_conf import GPSConf
from scs_dfe.interface.interface_conf import InterfaceConf
from scs_dfe.particulate.opc_conf import OPCConf

from scs_host.bus.i2c import I2C
from scs_host.sys.host import Host

try:
    from scs_ndir.gas.ndir.ndir_conf import NDIRConf
except ImportError:
    from scs_core.gas.ndir.ndir_conf import NDIRConf

# TODO: remove references to NDIRConf?
# --------------------------------------------------------------------------------------------------------------------

def power_gps(enable):
    if gps is None:
        interface.power_gps(enable)             # use interface if no GPS
        return

    if enable:
        gps.power_on()
    else:
        gps.power_off()

def power_ndir(enable):
    if ndir is None:
        interface.power_ndir(enable)             # use interface if no NDIR
        return

    if enable:
        ndir.power_on()
    else:
        ndir.power_off()

def power_opc(enable):
    if opc is None:
        interface.power_opc(enable)             # use interface if no OPC
        return

    if enable:
        opc.power_on()
        opc.operations_on()
    else:
        opc.operations_off()
        opc.power_off()

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    I2C.open(Host.I2C_SENSORS)

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdPower()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("interface_power: %s" % cmd, file=sys.stderr)

    try:
        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # Interface...
        interface_conf = InterfaceConf.load(Host)

        if interface_conf is None:
            print("interface_power: InterfaceConf not available.", file=sys.stderr)
            exit(1)

        interface = interface_conf.interface()

        if cmd.verbose:
            print("interface_power: %s" % interface, file=sys.stderr)
            sys.stderr.flush()

        # GPS...
        gps_conf = GPSConf.load(Host)
        gps = None if gps_conf is None else gps_conf.gps(interface, Host)

        if cmd.verbose and gps:
            print("interface_power: %s" % gps, file=sys.stderr)

        # NDIR...
        ndir_conf = NDIRConf.load(Host)
        ndir = None if ndir_conf is None else ndir_conf.ndir(interface, Host)

        if cmd.verbose and ndir:
            print("interface_power: %s" % ndir, file=sys.stderr)

        # OPC...
        opc_conf = OPCConf.load(Host)
        opc = None if opc_conf is None else opc_conf.opc(interface, Host)

        if cmd.verbose and opc:
            print("interface_power: %s" % opc, file=sys.stderr)

        # LED...
        led = interface.led()

        if cmd.verbose and led:
            print("interface_power: %s" % led, file=sys.stderr)

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("interface_power", cmd.verbose)

        if cmd.all is not None:
            interface.power_gases(cmd.all)
            interface.power_modem(cmd.all)
            power_gps(cmd.all)
            power_ndir(cmd.all)
            power_opc(cmd.all)

        if cmd.gases is not None:
            interface.power_gases(cmd.gases)

        if cmd.gps is not None:
            power_gps(cmd.gps)

        if cmd.modem is not None:
            interface.power_modem(cmd.modem)

        if cmd.ndir is not None:
            power_ndir(cmd.ndir)

        if cmd.opc is not None:
            power_opc(cmd.opc)

        if cmd.led is not None:
            if led:
                led.colour = cmd.led

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("interface_power: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("interface_power: finishing", file=sys.stderr)

        I2C.close()
<|endoftext|>


<commit_before>"""
seabreeze.spectrometers core module
===================================

Author: Andreas Poehlmann
Email: andreas@poehlmann.io

"""
import numpy

import seabreeze.backends
from seabreeze.compat import DeprecatedSpectrometerMixin as _DeprecatedSpectrometerMixin

# get the backend and add some functions/classes to this module
_lib = seabreeze.backends.get_backend()
_api = _lib.SeaBreezeAPI()

SeaBreezeError = _lib.SeaBreezeError
SeaBreezeDevice = _lib.SeaBreezeDevice

def list_devices():
    """returns available SeaBreezeDevices

    list all connected Ocean Optics devices supported

    Returns
    -------
    devices: `list[SeaBreezeDevice]`
        connected Spectrometer instances
    """
    return _api.list_devices()

class Spectrometer(_DeprecatedSpectrometerMixin):
    """Spectrometer class for all supported spectrometers"""

    def __init__(self, device):
        """create a Spectrometer instance for the provided device

        The Spectrometer class provides a thin abstraction layer for the
        basic spectrometer feature of the provided SeaBreezeDevice.

        Parameters
        ----------
        device : `seabreeze.spectrometers.SeaBreezeDevice`
            a SeaBreezeDevice as returned from `list_devices`
        """
        if not isinstance(device, SeaBreezeDevice):
            raise TypeError("device has to be a `SeaBreezeDevice`")
        self._dev = device
        self.open()  # always open the device here to allow caching values

        # check for nonlinearity correction support
        nc_feature = self._dev.f.nonlinearity_coefficients
        self._nc = None
        if nc_feature is not None:
            try:
                # NOTE: the spark spectrometer raises a transport error when trying
                # to receive the nc coefficients. In this case continue with disabled
                # nonlinearity correction support
                self._nc = nc_feature.get_nonlinearity_coefficients()
            except SeaBreezeError:
                pass
        # check for dark pixel correction support
        self._dp = self._dev.f.spectrometer.get_electric_dark_pixel_indices()
        # cache wavelengths on open
        self._wavelengths = self._dev.f.spectrometer.get_wavelengths()

    @classmethod
    def from_first_available(cls):
        """open first available spectrometer

        Returns
        -------
        spectrometer : `Spectrometer`
            the first available supported spectrometer
        """
        for dev in list_devices():
            if not dev.is_open:
                return cls(dev)
        else:
            raise SeaBreezeError("No unopened device found.")

    @classmethod
    def from_serial_number(cls, serial=None):
        """open the spectrometer matching the provided serial number

        Allows to open a specific spectrometer if multiple are connected.
        Spectrometer serial numbers are visible in the repr string of
        each `SeaBreezeDevice` or their `serial_number` attribute.

        Parameters
        ----------
        serial : `str`, optional
            the spectrometer's serial number. If `None` (default) it
            returns the first available unopened spectrometer.

        Returns
        -------
        spectrometer : `Spectrometer`
            the spectrometer with the requested serial number
        """
        if serial is None:  # pick first spectrometer
            return cls.from_first_available()

        for dev in list_devices():
            if dev.serial_number == str(serial):
                if dev.is_open:
                    raise SeaBreezeError("Device already opened.")
                else:
                    return cls(dev)
        else:
            raise SeaBreezeError("No device attached with serial number '%s'." % serial)

    def wavelengths(self):
        """wavelength array of the spectrometer

        wavelengths in (nm) corresponding to each pixel of the spectrometer

        Returns
        -------
        wavelengths : `numpy.ndarray`
            wavelengths in (nm)
        """
        return self._wavelengths

    def intensities(self, correct_dark_counts=False, correct_nonlinearity=False):
        """measured intensity array in (a.u.)

        Measured intensities as numpy array returned by the spectrometer.
        The measuring behavior can be adjusted by setting the trigger mode.
        Pixels at the start and end of the array might not be optically
        active so interpret their returned measurements with care. Refer
        to the spectrometer's datasheet for further information.

        Notes
        -----
        Intensities are in arbitrary units and the range depends on the
        ADC bit resolution of the hardware used in the specific spectrometer.
        Some spectrometers store a `saturation` value in their eeprom,
        which is used to rescale the raw ADC output to the full bit range.
        (This is done in `libseabreeze` and therefore also in `cseabreeze`
        --- for compatibility reasons the same is done in `pyseabreeze`)
        I.e. this means that a 16bit (max value 65535) spectrometer with a
        saturation value of ~30000 is effectively only returning ~15bit
        resolution raw readings. While most of the lower bits are dominated
        by noise anyways, it's just something to keep in mind. Refer to
        `pyseabreeze.features.spectrometer._SeaBreezeSpectrometerSaturationMixin`
        for the implementation.

        Parameters
        ----------
        correct_dark_counts : `bool`
            If requested and supported the average value of electric dark
            pixels on the ccd of the spectrometer is subtracted from the
            measurements to remove the noise floor in the measurements
            caused by non optical noise sources.
        correct_nonlinearity : `bool`
            Some spectrometers store non linearity correction coefficients
            in their eeprom. If requested and supported by the spectrometer
            the readings returned by the spectrometer will be linearized
            using the stored coefficients.

        Returns
        -------
        intensities : `numpy.ndarray`
            measured intensities in (a.u.)
        """
        if correct_dark_counts and not self._dp:
            raise SeaBreezeError("This device does not support dark count correction.")
        if correct_nonlinearity and not self._nc:
            raise SeaBreezeError("This device does not support nonlinearity correction.")
        # Get the intensities
        out = self._dev.f.spectrometer.get_intensities()
        # Do corrections if requested
        if correct_nonlinearity or correct_dark_counts:
            dark_offset = numpy.mean(out[self._dp]) if self._dp else 0.
            out -= dark_offset
        if correct_nonlinearity:
            out = out / numpy.polyval(self._nc, out)
        if correct_nonlinearity and (not correct_dark_counts):
            # noinspection PyUnboundLocalVariable
            out += dark_offset
        return out

    @property
    def max_intensity(self):
        """return the maximum intensity of the spectrometer

        Returns
        -------
        max_intensity : `float`
            the maximum intensity that can be returned by the spectrometer in (a.u.)
            It's possible that the spectrometer saturates already at lower values.
        """
        return self._dev.f.spectrometer.get_maximum_intensity()

    def spectrum(self, correct_dark_counts=False, correct_nonlinearity=False):
        """returns wavelengths and intensities as single array

        Convenience method to allow:

        >>> spec = Spectrometer.from_first_available()
        >>> wavelengths, intensities = spec.spectrum()

        Parameters
        ----------
        correct_dark_counts : `bool`
            see `Spectrometer.intensities`
        correct_nonlinearity : `bool`
            see `Spectrometer.intensities`

        Returns
        -------
        spectrum : `numpy.ndarray`
            combined array of wavelengths and measured intensities
        """
        return numpy.vstack((self._wavelengths,
                             self.intensities(correct_dark_counts, correct_nonlinearity)))

    def integration_time_micros(self, integration_time_micros):
        """set the integration time in microseconds

        Parameters
        ----------
        integration_time_micros : `int`
            integration time in microseconds
        """
        # NOTE: Protect against a bug in libseabreeze:
        # If integration time is out of bounds, libseabreeze returns Undefined Error
        # (Probably only for devices with a non micro second time base...)
        try:
            self._dev.f.spectrometer.set_integration_time_micros(integration_time_micros)
        except SeaBreezeError as e:
            if getattr(e, 'error_code', None) == 1:
                # Only replace if 'Undefined Error'
                raise SeaBreezeError("FIX: Specified integration time is out of range.")
            else:
                raise e

    @property
    def integration_time_micros_limits(self):
        """return the hardcoded minimum and maximum integration time

        Returns
        -------
        integration_time_micros_min_max : `tuple[int, int]`
            min and max integration time in micro seconds
        """
        return self._dev.f.spectrometer.get_integration_time_micros_limits()

    def trigger_mode(self, mode):
        """set the trigger mode of the device

        Parameters
        ----------
        mode : `int`
            refer to your spectrometer's datasheet to determine the correct
            value for the trigger mode you want to use.
        """
        self._dev.f.spectrometer.set_trigger_mode(mode)

    @property
    def serial_number(self):
        """the spectrometer's serial number"""
        return self._dev.serial_number

    @property
    def model(self):
        """the spectrometer's model type"""
        return self._dev.model

    @property
    def pixels(self):
        """the spectrometer's number of pixels"""
        # noinspection PyProtectedMember
        return self._dev.f.spectrometer._spectrum_length

    def open(self):
        """open the connection to the SeaBreezeDevice

        Notes
        -----
        Normally you do not have to call this function manually.
        If you're trying to use the same spectrometer from multiple
        processes (honestly, why would you?) then this might come
        in handy. But I'll leave the lock/semaphore handling to you.
        """
        self._dev.open()

    def close(self):
        """close the connection to the SeaBreezeDevice

        Notes
        -----
        Normally you do not have to call this function manually.
        If you're trying to use the same spectrometer from multiple
        processes (honestly, why would you?) then this might come
        in handy. But I'll leave the lock/semaphore handling to you.
        """
        self._dev.close()

    def __repr__(self):
        return "<Spectrometer %s:%s>" % (self.model, self.serial_number)
<commit_msg>seabreeze.spectrometers: provide access to all backend feature classes<commit_after>"""
seabreeze.spectrometers core module
===================================

Author: Andreas Poehlmann
Email: andreas@poehlmann.io

"""
import numpy

import seabreeze.backends
from seabreeze.compat import DeprecatedSpectrometerMixin as _DeprecatedSpectrometerMixin

# get the backend and add some functions/classes to this module
_lib = seabreeze.backends.get_backend()
_api = _lib.SeaBreezeAPI()

SeaBreezeError = _lib.SeaBreezeError
SeaBreezeDevice = _lib.SeaBreezeDevice
SeaBreezeFeature = _lib.SeaBreezeFeature

def list_devices():
    """returns available SeaBreezeDevices

    list all connected Ocean Optics devices supported

    Returns
    -------
    devices: `list[SeaBreezeDevice]`
        connected Spectrometer instances
    """
    return _api.list_devices()

class Spectrometer(_DeprecatedSpectrometerMixin):
    """Spectrometer class for all supported spectrometers"""

    def __init__(self, device):
        """create a Spectrometer instance for the provided device

        The Spectrometer class provides a thin abstraction layer for the
        basic spectrometer feature of the provided SeaBreezeDevice.

        Parameters
        ----------
        device : `seabreeze.spectrometers.SeaBreezeDevice`
            a SeaBreezeDevice as returned from `list_devices`
        """
        if not isinstance(device, SeaBreezeDevice):
            raise TypeError("device has to be a `SeaBreezeDevice`")
        self._dev = device
        self.open()  # always open the device here to allow caching values

        # check for nonlinearity correction support
        nc_feature = self._dev.f.nonlinearity_coefficients
        self._nc = None
        if nc_feature is not None:
            try:
                # NOTE: the spark spectrometer raises a transport error when trying
                # to receive the nc coefficients. In this case continue with disabled
                # nonlinearity correction support
                self._nc = nc_feature.get_nonlinearity_coefficients()
            except SeaBreezeError:
                pass
        # check for dark pixel correction support
        self._dp = self._dev.f.spectrometer.get_electric_dark_pixel_indices()
        # cache wavelengths on open
        self._wavelengths = self._dev.f.spectrometer.get_wavelengths()

    @classmethod
    def from_first_available(cls):
        """open first available spectrometer

        Returns
        -------
        spectrometer : `Spectrometer`
            the first available supported spectrometer
        """
        for dev in list_devices():
            if not dev.is_open:
                return cls(dev)
        else:
            raise SeaBreezeError("No unopened device found.")

    @classmethod
    def from_serial_number(cls, serial=None):
        """open the spectrometer matching the provided serial number

        Allows to open a specific spectrometer if multiple are connected.
        Spectrometer serial numbers are visible in the repr string of
        each `SeaBreezeDevice` or their `serial_number` attribute.

        Parameters
        ----------
        serial : `str`, optional
            the spectrometer's serial number. If `None` (default) it
            returns the first available unopened spectrometer.

        Returns
        -------
        spectrometer : `Spectrometer`
            the spectrometer with the requested serial number
        """
        if serial is None:  # pick first spectrometer
            return cls.from_first_available()

        for dev in list_devices():
            if dev.serial_number == str(serial):
                if dev.is_open:
                    raise SeaBreezeError("Device already opened.")
                else:
                    return cls(dev)
        else:
            raise SeaBreezeError("No device attached with serial number '%s'." % serial)

    def wavelengths(self):
        """wavelength array of the spectrometer

        wavelengths in (nm) corresponding to each pixel of the spectrometer

        Returns
        -------
        wavelengths : `numpy.ndarray`
            wavelengths in (nm)
        """
        return self._wavelengths

    def intensities(self, correct_dark_counts=False, correct_nonlinearity=False):
        """measured intensity array in (a.u.)

        Measured intensities as numpy array returned by the spectrometer.
        The measuring behavior can be adjusted by setting the trigger mode.
        Pixels at the start and end of the array might not be optically
        active so interpret their returned measurements with care. Refer
        to the spectrometer's datasheet for further information.

        Notes
        -----
        Intensities are in arbitrary units and the range depends on the
        ADC bit resolution of the hardware used in the specific spectrometer.
        Some spectrometers store a `saturation` value in their eeprom,
        which is used to rescale the raw ADC output to the full bit range.
        (This is done in `libseabreeze` and therefore also in `cseabreeze`
        --- for compatibility reasons the same is done in `pyseabreeze`)
        I.e. this means that a 16bit (max value 65535) spectrometer with a
        saturation value of ~30000 is effectively only returning ~15bit
        resolution raw readings. While most of the lower bits are dominated
        by noise anyways, it's just something to keep in mind. Refer to
        `pyseabreeze.features.spectrometer._SeaBreezeSpectrometerSaturationMixin`
        for the implementation.

        Parameters
        ----------
        correct_dark_counts : `bool`
            If requested and supported the average value of electric dark
            pixels on the ccd of the spectrometer is subtracted from the
            measurements to remove the noise floor in the measurements
            caused by non optical noise sources.
        correct_nonlinearity : `bool`
            Some spectrometers store non linearity correction coefficients
            in their eeprom. If requested and supported by the spectrometer
            the readings returned by the spectrometer will be linearized
            using the stored coefficients.

        Returns
        -------
        intensities : `numpy.ndarray`
            measured intensities in (a.u.)
        """
        if correct_dark_counts and not self._dp:
            raise SeaBreezeError("This device does not support dark count correction.")
        if correct_nonlinearity and not self._nc:
            raise SeaBreezeError("This device does not support nonlinearity correction.")
        # Get the intensities
        out = self._dev.f.spectrometer.get_intensities()
        # Do corrections if requested
        if correct_nonlinearity or correct_dark_counts:
            dark_offset = numpy.mean(out[self._dp]) if self._dp else 0.
            out -= dark_offset
        if correct_nonlinearity:
            out = out / numpy.polyval(self._nc, out)
        if correct_nonlinearity and (not correct_dark_counts):
            # noinspection PyUnboundLocalVariable
            out += dark_offset
        return out

    @property
    def max_intensity(self):
        """return the maximum intensity of the spectrometer

        Returns
        -------
        max_intensity : `float`
            the maximum intensity that can be returned by the spectrometer in (a.u.)
            It's possible that the spectrometer saturates already at lower values.
        """
        return self._dev.f.spectrometer.get_maximum_intensity()

    def spectrum(self, correct_dark_counts=False, correct_nonlinearity=False):
        """returns wavelengths and intensities as single array

        Convenience method to allow:

        >>> spec = Spectrometer.from_first_available()
        >>> wavelengths, intensities = spec.spectrum()

        Parameters
        ----------
        correct_dark_counts : `bool`
            see `Spectrometer.intensities`
        correct_nonlinearity : `bool`
            see `Spectrometer.intensities`

        Returns
        -------
        spectrum : `numpy.ndarray`
            combined array of wavelengths and measured intensities
        """
        return numpy.vstack((self._wavelengths,
                             self.intensities(correct_dark_counts, correct_nonlinearity)))

    def integration_time_micros(self, integration_time_micros):
        """set the integration time in microseconds

        Parameters
        ----------
        integration_time_micros : `int`
            integration time in microseconds
        """
        # NOTE: Protect against a bug in libseabreeze:
        # If integration time is out of bounds, libseabreeze returns Undefined Error
        # (Probably only for devices with a non micro second time base...)
        try:
            self._dev.f.spectrometer.set_integration_time_micros(integration_time_micros)
        except SeaBreezeError as e:
            if getattr(e, 'error_code', None) == 1:
                # Only replace if 'Undefined Error'
                raise SeaBreezeError("FIX: Specified integration time is out of range.")
            else:
                raise e

    @property
    def integration_time_micros_limits(self):
        """return the hardcoded minimum and maximum integration time

        Returns
        -------
        integration_time_micros_min_max : `tuple[int, int]`
            min and max integration time in micro seconds
        """
        return self._dev.f.spectrometer.get_integration_time_micros_limits()

    def trigger_mode(self, mode):
        """set the trigger mode of the device

        Parameters
        ----------
        mode : `int`
            refer to your spectrometer's datasheet to determine the correct
            value for the trigger mode you want to use.
        """
        self._dev.f.spectrometer.set_trigger_mode(mode)

    @property
    def serial_number(self):
        """the spectrometer's serial number"""
        return self._dev.serial_number

    @property
    def model(self):
        """the spectrometer's model type"""
        return self._dev.model

    @property
    def pixels(self):
        """the spectrometer's number of pixels"""
        # noinspection PyProtectedMember
        return self._dev.f.spectrometer._spectrum_length

    @property
    def features(self):
        """return a dictionary of all supported features

        this returns a dictionary with all supported Features of the spectrometer
        and gives direct access to the features provided by the backend interface.

        Returns
        -------
        features : `dict` [`str`, `seabreeze.SeaBreezeFeature`]
        """
        return self._dev.features

    @property
    def f(self):
        """convenience assess to features via attributes

        this allows you to access a feature like this::

        >>> spec = Spectrometer.from_first_available()
        >>> # via .features
        >>> spec.features['eeprom'][0].eeprom_read_slot(4)
        >>> # via .f
        >>> spec.f.eeprom.eeprom_read_slot(4)

        """
        return self._dev.f

    def open(self):
        """open the connection to the SeaBreezeDevice

        Notes
        -----
        Normally you do not have to call this function manually.
        If you're trying to use the same spectrometer from multiple
        processes (honestly, why would you?) then this might come
        in handy. But I'll leave the lock/semaphore handling to you.
        """
        self._dev.open()

    def close(self):
        """close the connection to the SeaBreezeDevice

        Notes
        -----
        Normally you do not have to call this function manually.
        If you're trying to use the same spectrometer from multiple
        processes (honestly, why would you?) then this might come
        in handy. But I'll leave the lock/semaphore handling to you.
        """
        self._dev.close()

    def __repr__(self):
        return "<Spectrometer %s:%s>" % (self.model, self.serial_number)
<|endoftext|>