<commit_before>from .oop import (GrpcMethod, OopProxy, transform_map, mirror_enum, unwrap)
from .proto import (xenon_pb2, xenon_pb2_grpc)
from .server import __server__

import pathlib
import inspect
import functools

try:
    from os import PathLike
except ImportError:
    PathLike = object

CopyMode = mirror_enum(xenon_pb2.CopyRequest, 'CopyMode')
PosixFilePermission = mirror_enum(xenon_pb2, 'PosixFilePermission')
Type = mirror_enum(xenon_pb2.PropertyDescription, 'Type')

class CopyStatus(OopProxy):
    """Status of a copy operation."""
    ErrorType = mirror_enum(xenon_pb2.CopyStatus, 'ErrorType')

    @property
    def error_type(self):
        return CopyStatus.ErrorType(self.__wrapped__.error_type)

class JobStatus(OopProxy):
    """Status of a job."""
    ErrorType = mirror_enum(xenon_pb2.JobStatus, 'ErrorType')

    @property
    def error_type(self):
        return JobStatus.ErrorType(self.__wrapped__.error_type)

class QueueStatus(OopProxy):
    """Status of a queue."""
    ErrorType = mirror_enum(xenon_pb2.QueueStatus, 'ErrorType')

    @property
    def error_type(self):
        return QueueStatus.ErrorType(self.__wrapped__.error_type)

class PathAttributes(OopProxy):
    def __init__(self, service, wrapped):
        super(PathAttributes, self).__init__(service, wrapped)

    @property
    def path(self):
        return Path(self.__wrapped__.path.path)

class CopyOperation(OopProxy):
    pass

class Job(object):
    """Job.

    :ivar id: the Xenon job identifyer.
    :vartype id: string
    """
    __is_proxy__ = True
    __servicer__ = None

    def __init__(self, id_):
        self.id = id_

    @property
    def __wrapped__(self):
        return xenon_pb2.Job(id=self.id)

class Is(OopProxy):
    def __bool__(self):
        return self.value

def append_request_stream(self, path, data_stream):
    yield xenon_pb2.AppendToFileRequest(
        filesystem=unwrap(self), path=unwrap(path))
    yield from (xenon_pb2.AppendToFileRequest(buffer=b)
                for b in data_stream)

def write_request_stream(self, path, data_stream):
    yield xenon_pb2.WriteToFileRequest(
        filesystem=unwrap(self), path=unwrap(path))
    yield from (xenon_pb2.WriteToFileRequest(buffer=b)
                for b in data_stream)

class Path(PathLike):
    """Wrapper around :py:class:`PurePosixPath` form the :py:mod:`pathlib`
    module.  This class reveals a string representation of the underlying path
    object to GRPC. You may use this class like a `pathlib.PurePosixPath`,
    including using it as an argument to `open` calls as it derives from
    `os.PathLike` (Python > 3.6). For more information see `the Python
    documentation on pathlib
    <https://docs.python.org/3/library/pathlib.html>`_."""
    __is_proxy__ = True
    __servicer__ = xenon_pb2_grpc.FileSystemServiceServicer

    def __init__(self, path):
        if isinstance(path, pathlib.PurePosixPath):
            self._pathlib_path = path
        elif isinstance(path, xenon_pb2.Path):
            self._pathlib_path = pathlib.PurePosixPath(path.path)
        else:
            self._pathlib_path = pathlib.PurePosixPath(path)

    def __str__(self):
        return str(self._pathlib_path)

    @property
    def __wrapped__(self):
        return xenon_pb2.Path(
            path=str(self._pathlib_path),  # .__fspath__(),
            separator='/')

    def __truediv__(self, other):
        return Path(self._pathlib_path / other)

    def __fspath__(self):
        return self._pathlib_path.__fspath__()

    def __getattr__(self, attr):
        if attr == '__wrapped__':
            print("Warning: faulty Python behaviour.")
            return xenon_pb2.Path(
                path=str(self._pathlib_path),  # .__fspath__(),
                separator='/')

        member = getattr(self._pathlib_path, attr)
        if inspect.ismethod(member):
            @functools.wraps(member)
            def wrapped_member(*args, **kwargs):
                value = member(*args, **kwargs)
                if isinstance(value, pathlib.PurePosixPath):
                    return Path(value)
                else:
                    return value

            return wrapped_member

        elif isinstance(member, pathlib.PurePosixPath):
            return Path(member)

        else:
            return member

    def __dir__(self):
        return list(set(dir(self._pathlib_path) + dir(self)))

    def is_hidden(self):
        """Checks if a file is hidden. Just compares the first character in the
        filename with `'.'`."""
        return self.name[0] == '.'

def t_getattr(name):
    return lambda self, x: getattr(x, name)

def read_response_stream(self, stream):
    yield from (chunk.buffer for chunk in stream)

class FileSystem(OopProxy):
    """The Xenon `FileSystem` subsystem."""
    __servicer__ = xenon_pb2_grpc.FileSystemServiceServicer
    __field_name__ = 'filesystem'

    @classmethod
    def __methods__(cls):
        return [
            GrpcMethod(
                'get_adaptor_descriptions', static=True,
                output_transform=t_getattr('descriptions')),
            GrpcMethod(
                'get_adaptor_names', static=True,
                output_transform=t_getattr('name')),
            GrpcMethod(
                'get_adaptor_description', uses_request='AdaptorName',
                static=True),
            GrpcMethod(
                'create', static=True,
                uses_request='CreateFileSystemRequest',
                output_transform=cls),

            GrpcMethod(
                'local_file_systems', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.filesystems)),
            GrpcMethod(
                'list_file_systems', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.filesystems)),

            GrpcMethod(
                'get_adaptor_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'rename', uses_request=True),
            GrpcMethod(
                'create_symbolic_link', uses_request=True),
            GrpcMethod(
                'get_working_directory',
                output_transform=lambda self, x: Path(x)),
            GrpcMethod(
                'set_working_directory', uses_request='PathRequest'),
            GrpcMethod(
                'is_open', output_transform=Is),
            GrpcMethod(
                'close'),
            GrpcMethod(
                'cancel', uses_request='CopyOperationRequest',
                output_transform=CopyStatus),
            GrpcMethod(
                'get_status', uses_request='CopyOperationRequest',
                output_transform=CopyStatus),
            GrpcMethod(
                'wait_until_done', uses_request=True,
                output_transform=CopyStatus),
            GrpcMethod(
                'create_directories', uses_request='PathRequest'),
            GrpcMethod(
                'create_directory', uses_request='PathRequest'),
            GrpcMethod(
                'create_file', uses_request='PathRequest'),
            GrpcMethod(
                'exists', uses_request='PathRequest',
                output_transform=Is),
            GrpcMethod(
                'read_from_file', uses_request='PathRequest',
                output_transform=read_response_stream),
            GrpcMethod(
                'get_attributes', uses_request='PathRequest',
                output_transform=PathAttributes),
            GrpcMethod(
                'read_symbolic_link', uses_request='PathRequest',
                output_transform=lambda self, x: Path(x)),
            GrpcMethod(
                'write_to_file', input_transform=write_request_stream),
            GrpcMethod(
                'append_to_file', input_transform=append_request_stream),
            GrpcMethod(
                'delete', uses_request=True),
            GrpcMethod(
                'copy', uses_request=True, output_transform=CopyOperation),
            GrpcMethod(
                'set_posix_file_permissions', uses_request=True),
            GrpcMethod(
                'list', uses_request=True,
                output_transform=transform_map(PathAttributes)),

            GrpcMethod(
                'get_path_separator', output_transform=t_getattr('separator'))
        ]

    @staticmethod
    def __stub__(server):
        return server.file_system_stub

    def __init__(self, service, wrapped):
        super(FileSystem, self).__init__(service, wrapped)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

    def __eq__(self, other):
        return self.__wrapped__ == other.__wrapped__

def input_request_stream(self, description, stdin_stream):
    yield xenon_pb2.SubmitInteractiveJobRequest(
        scheduler=unwrap(self), description=description, stdin=b'')
    yield from (xenon_pb2.SubmitInteractiveJobRequest(
        scheduler=None, description=None, stdin=msg)
        for msg in stdin_stream)

def interactive_job_response(self, stream):
    job = stream.next().job
    return job, stream

class Scheduler(OopProxy):
    """The Xenon Schedulers subsystem."""
    __servicer__ = xenon_pb2_grpc.SchedulerServiceServicer
    __field_name__ = 'scheduler'

    @classmethod
    def __methods__(cls):
        return [
            GrpcMethod(
                'local_scheduler', static=True, output_transform=cls),
            GrpcMethod(
                'list_schedulers', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.schedulers)),

            GrpcMethod(
                'get_adaptor_descriptions', static=True,
                output_transform=t_getattr('descriptions')),
            GrpcMethod(
                'get_adaptor_names', static=True),
            GrpcMethod(
                'get_adaptor_description', static=True,
                uses_request='AdaptorName'),
            GrpcMethod(
                'create', static=True, uses_request='CreateSchedulerRequest',
                output_transform=cls),

            GrpcMethod(
                'get_adaptor_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'get_location', output_transform=t_getattr('location')),
            GrpcMethod(
                'get_properties', output_transform=t_getattr('properties')),
            GrpcMethod(
                'get_jobs', uses_request='SchedulerAndQueues',
                output_transform=t_getattr('jobs')),
            GrpcMethod(
                'get_queue_names', output_transform=t_getattr('name')),
            GrpcMethod(
                'get_default_queue_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'is_open', output_transform=t_getattr('value')),
            GrpcMethod(
                'close'),
            GrpcMethod(
                'submit_batch_job', uses_request=True,
                output_transform=lambda s, x: Job(x.id)),
            GrpcMethod(
                'submit_interactive_job', input_transform=input_request_stream,
                output_transform=interactive_job_response),
            GrpcMethod(
                'cancel_job', uses_request='JobRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'wait_until_done', uses_request='WaitRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'wait_until_running', uses_request='WaitRequest',
                output_transform=JobStatus),

            GrpcMethod(
                'get_queue_status', uses_request=True),
            GrpcMethod(
                'get_queue_statuses', uses_request='SchedulerAndQueues',
                output_transform=t_getattr('statuses')),

            GrpcMethod(
                'get_job_status', uses_request='JobRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'get_job_statuses', uses_request=True,
                output_transform=lambda s, x:
                    [JobStatus(s, j) for j in x.statuses]),

            # smells like tenenkaas
            GrpcMethod(
                'get_file_system',
                output_transform=lambda s, x:
                    FileSystem(FileSystem.__stub__(__server__), x))
        ]

    def __init__(self, service, wrapped):
        super(Scheduler, self).__init__(service, wrapped)

    @staticmethod
    def __stub__(server):
        return server.scheduler_stub

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()
<commit_msg>wrap JobDescription, so that Paths are handled for the `working_directory` attribute<commit_after>from .oop import (
    GrpcMethod, OopProxy, transform_map, mirror_enum, unwrap)

from .proto import (xenon_pb2, xenon_pb2_grpc)
from .server import __server__

import pathlib
import inspect
import functools

try:
    from os import PathLike
except ImportError:
    PathLike = object

CopyMode = mirror_enum(xenon_pb2.CopyRequest, 'CopyMode')
PosixFilePermission = mirror_enum(xenon_pb2, 'PosixFilePermission')
Type = mirror_enum(xenon_pb2.PropertyDescription, 'Type')

class CopyStatus(OopProxy):
    """Status of a copy operation."""
    ErrorType = mirror_enum(xenon_pb2.CopyStatus, 'ErrorType')

    @property
    def error_type(self):
        return CopyStatus.ErrorType(self.__wrapped__.error_type)

class JobStatus(OopProxy):
    """Status of a job."""
    ErrorType = mirror_enum(xenon_pb2.JobStatus, 'ErrorType')

    @property
    def error_type(self):
        return JobStatus.ErrorType(self.__wrapped__.error_type)

class QueueStatus(OopProxy):
    """Status of a queue."""
    ErrorType = mirror_enum(xenon_pb2.QueueStatus, 'ErrorType')

    @property
    def error_type(self):
        return QueueStatus.ErrorType(self.__wrapped__.error_type)

class PathAttributes(OopProxy):
    def __init__(self, service, wrapped):
        super(PathAttributes, self).__init__(service, wrapped)

    @property
    def path(self):
        return Path(self.__wrapped__.path.path)

class CopyOperation(OopProxy):
    pass

class JobDescription(object):
    __is_proxy__ = True
    __servicer__ = None

    def __init__(self, **kwargs):
        self.__wrapped__ = xenon_pb2.JobDescription()
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __setattr__(self, attr, value):
        setattr(self.__wrapped__, attr, value)

    def __getattr__(self, attr):
        return getattr(self.__wrapped__, attr)

    @property
    def working_directory(self):
        return Path(self.__wrapped__.working_directory)

    @working_directory.setter
    def working_directory(self, value):
        self.__wrapped__.working_directory = str(value)

class Job(object):
    """Job.

    :ivar id: the Xenon job identifyer.
    :vartype id: string
    """
    __is_proxy__ = True
    __servicer__ = None

    def __init__(self, id_):
        self.id = id_

    @property
    def __wrapped__(self):
        return xenon_pb2.Job(id=self.id)

class Is(OopProxy):
    def __bool__(self):
        return self.value

def append_request_stream(self, path, data_stream):
    yield xenon_pb2.AppendToFileRequest(
        filesystem=unwrap(self), path=unwrap(path))
    yield from (xenon_pb2.AppendToFileRequest(buffer=b)
                for b in data_stream)

def write_request_stream(self, path, data_stream):
    yield xenon_pb2.WriteToFileRequest(
        filesystem=unwrap(self), path=unwrap(path))
    yield from (xenon_pb2.WriteToFileRequest(buffer=b)
                for b in data_stream)

class Path(PathLike):
    """Wrapper around :py:class:`PurePosixPath` form the :py:mod:`pathlib`
    module.  This class reveals a string representation of the underlying path
    object to GRPC. You may use this class like a `pathlib.PurePosixPath`,
    including using it as an argument to `open` calls as it derives from
    `os.PathLike` (Python > 3.6). For more information see `the Python
    documentation on pathlib
    <https://docs.python.org/3/library/pathlib.html>`_."""
    __is_proxy__ = True
    __servicer__ = xenon_pb2_grpc.FileSystemServiceServicer

    def __init__(self, path):
        if isinstance(path, pathlib.PurePosixPath):
            self._pathlib_path = path
        elif isinstance(path, xenon_pb2.Path):
            self._pathlib_path = pathlib.PurePosixPath(path.path)
        else:
            self._pathlib_path = pathlib.PurePosixPath(path)

    def __str__(self):
        return str(self._pathlib_path)

    @property
    def __wrapped__(self):
        return xenon_pb2.Path(
            path=str(self._pathlib_path),  # .__fspath__(),
            separator='/')

    def __truediv__(self, other):
        return Path(self._pathlib_path / other)

    def __fspath__(self):
        return self._pathlib_path.__fspath__()

    def __getattr__(self, attr):
        if attr == '__wrapped__':
            print("Warning: faulty Python behaviour.")
            return xenon_pb2.Path(
                path=str(self._pathlib_path),  # .__fspath__(),
                separator='/')

        member = getattr(self._pathlib_path, attr)
        if inspect.ismethod(member):
            @functools.wraps(member)
            def wrapped_member(*args, **kwargs):
                value = member(*args, **kwargs)
                if isinstance(value, pathlib.PurePosixPath):
                    return Path(value)
                else:
                    return value

            return wrapped_member

        elif isinstance(member, pathlib.PurePosixPath):
            return Path(member)

        else:
            return member

    def __dir__(self):
        return list(set(dir(self._pathlib_path) + dir(self)))

    def is_hidden(self):
        """Checks if a file is hidden. Just compares the first character in the
        filename with `'.'`."""
        return self.name[0] == '.'

def t_getattr(name):
    return lambda self, x: getattr(x, name)

def read_response_stream(self, stream):
    yield from (chunk.buffer for chunk in stream)

class FileSystem(OopProxy):
    """The Xenon `FileSystem` subsystem."""
    __servicer__ = xenon_pb2_grpc.FileSystemServiceServicer
    __field_name__ = 'filesystem'

    @classmethod
    def __methods__(cls):
        return [
            GrpcMethod(
                'get_adaptor_descriptions', static=True,
                output_transform=t_getattr('descriptions')),
            GrpcMethod(
                'get_adaptor_names', static=True,
                output_transform=t_getattr('name')),
            GrpcMethod(
                'get_adaptor_description', uses_request='AdaptorName',
                static=True),
            GrpcMethod(
                'create', static=True,
                uses_request='CreateFileSystemRequest',
                output_transform=cls),

            GrpcMethod(
                'local_file_systems', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.filesystems)),
            GrpcMethod(
                'list_file_systems', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.filesystems)),

            GrpcMethod(
                'get_adaptor_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'rename', uses_request=True),
            GrpcMethod(
                'create_symbolic_link', uses_request=True),
            GrpcMethod(
                'get_working_directory',
                output_transform=lambda self, x: Path(x)),
            GrpcMethod(
                'set_working_directory', uses_request='PathRequest'),
            GrpcMethod(
                'is_open', output_transform=Is),
            GrpcMethod(
                'close'),
            GrpcMethod(
                'cancel', uses_request='CopyOperationRequest',
                output_transform=CopyStatus),
            GrpcMethod(
                'get_status', uses_request='CopyOperationRequest',
                output_transform=CopyStatus),
            GrpcMethod(
                'wait_until_done', uses_request=True,
                output_transform=CopyStatus),
            GrpcMethod(
                'create_directories', uses_request='PathRequest'),
            GrpcMethod(
                'create_directory', uses_request='PathRequest'),
            GrpcMethod(
                'create_file', uses_request='PathRequest'),
            GrpcMethod(
                'exists', uses_request='PathRequest',
                output_transform=Is),
            GrpcMethod(
                'read_from_file', uses_request='PathRequest',
                output_transform=read_response_stream),
            GrpcMethod(
                'get_attributes', uses_request='PathRequest',
                output_transform=PathAttributes),
            GrpcMethod(
                'read_symbolic_link', uses_request='PathRequest',
                output_transform=lambda self, x: Path(x)),
            GrpcMethod(
                'write_to_file', input_transform=write_request_stream),
            GrpcMethod(
                'append_to_file', input_transform=append_request_stream),
            GrpcMethod(
                'delete', uses_request=True),
            GrpcMethod(
                'copy', uses_request=True, output_transform=CopyOperation),
            GrpcMethod(
                'set_posix_file_permissions', uses_request=True),
            GrpcMethod(
                'list', uses_request=True,
                output_transform=transform_map(PathAttributes)),

            GrpcMethod(
                'get_path_separator', output_transform=t_getattr('separator'))
        ]

    @staticmethod
    def __stub__(server):
        return server.file_system_stub

    def __init__(self, service, wrapped):
        super(FileSystem, self).__init__(service, wrapped)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

    def __eq__(self, other):
        return self.__wrapped__ == other.__wrapped__

def input_request_stream(self, description, stdin_stream):
    yield xenon_pb2.SubmitInteractiveJobRequest(
        scheduler=unwrap(self), description=description, stdin=b'')
    yield from (xenon_pb2.SubmitInteractiveJobRequest(
        scheduler=None, description=None, stdin=msg)
        for msg in stdin_stream)

def interactive_job_response(self, stream):
    job = stream.next().job
    return job, stream

class Scheduler(OopProxy):
    """The Xenon Schedulers subsystem."""
    __servicer__ = xenon_pb2_grpc.SchedulerServiceServicer
    __field_name__ = 'scheduler'

    @classmethod
    def __methods__(cls):
        return [
            GrpcMethod(
                'local_scheduler', static=True, output_transform=cls),
            GrpcMethod(
                'list_schedulers', static=True,
                output_transform=lambda s, xs:
                    (cls(s, x) for x in xs.schedulers)),

            GrpcMethod(
                'get_adaptor_descriptions', static=True,
                output_transform=t_getattr('descriptions')),
            GrpcMethod(
                'get_adaptor_names', static=True),
            GrpcMethod(
                'get_adaptor_description', static=True,
                uses_request='AdaptorName'),
            GrpcMethod(
                'create', static=True, uses_request='CreateSchedulerRequest',
                output_transform=cls),

            GrpcMethod(
                'get_adaptor_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'get_location', output_transform=t_getattr('location')),
            GrpcMethod(
                'get_properties', output_transform=t_getattr('properties')),
            GrpcMethod(
                'get_jobs', uses_request='SchedulerAndQueues',
                output_transform=t_getattr('jobs')),
            GrpcMethod(
                'get_queue_names', output_transform=t_getattr('name')),
            GrpcMethod(
                'get_default_queue_name', output_transform=t_getattr('name')),
            GrpcMethod(
                'is_open', output_transform=t_getattr('value')),
            GrpcMethod(
                'close'),
            GrpcMethod(
                'submit_batch_job', uses_request=True,
                output_transform=lambda s, x: Job(x.id)),
            GrpcMethod(
                'submit_interactive_job', input_transform=input_request_stream,
                output_transform=interactive_job_response),
            GrpcMethod(
                'cancel_job', uses_request='JobRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'wait_until_done', uses_request='WaitRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'wait_until_running', uses_request='WaitRequest',
                output_transform=JobStatus),

            GrpcMethod(
                'get_queue_status', uses_request=True),
            GrpcMethod(
                'get_queue_statuses', uses_request='SchedulerAndQueues',
                output_transform=t_getattr('statuses')),

            GrpcMethod(
                'get_job_status', uses_request='JobRequest',
                output_transform=JobStatus),
            GrpcMethod(
                'get_job_statuses', uses_request=True,
                output_transform=lambda s, x:
                    [JobStatus(s, j) for j in x.statuses]),

            # smells like tenenkaas
            GrpcMethod(
                'get_file_system',
                output_transform=lambda s, x:
                    FileSystem(FileSystem.__stub__(__server__), x))
        ]

    def __init__(self, service, wrapped):
        super(Scheduler, self).__init__(service, wrapped)

    @staticmethod
    def __stub__(server):
        return server.scheduler_stub

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
NAPALM Grains
=============

:codeauthor: Mircea Ulinic <mircea@cloudflare.com>
:maturity:   new
:depends:    napalm
:platform:   unix

Dependencies
------------

- :mod:`NAPALM proxy module <salt.proxies.napalm>`

.. versionadded:: 2016.11.0
'''

from __future__ import absolute_import

import logging
log = logging.getLogger(__name__)

# Salt lib
import salt.utils
import salt.utils.napalm

# ----------------------------------------------------------------------------------------------------------------------
# grains properties
# ----------------------------------------------------------------------------------------------------------------------

__virtualname__ = 'napalm'
__proxyenabled__ = ['napalm']

# ----------------------------------------------------------------------------------------------------------------------
# global variables
# ----------------------------------------------------------------------------------------------------------------------

GRAINS_CACHE = {}
DEVICE_CACHE = {}

# ----------------------------------------------------------------------------------------------------------------------
# property functions
# ----------------------------------------------------------------------------------------------------------------------

def __virtual__():
    return __virtualname__

# ----------------------------------------------------------------------------------------------------------------------
# helpers
# ----------------------------------------------------------------------------------------------------------------------

def _retrieve_grains(proxy=None):
    '''
    Retrieves the grains from the network device if not cached already.
    '''

    global GRAINS_CACHE
    global DEVICE_CACHE

    if not GRAINS_CACHE:
        if proxy and salt.utils.napalm.is_proxy():
            # if proxy var passed and is NAPALM-type proxy minion
            GRAINS_CACHE = proxy['napalm.get_grains']()
            if 'napalm.get_device' in proxy:
                DEVICE_CACHE = proxy['napalm.get_device']()
        elif not proxy and salt.utils.napalm.is_minion():
            # if proxy var not passed and is running in a straight minion
            DEVICE_CACHE = salt.utils.napalm.get_device(__opts__)
            GRAINS_CACHE = salt.utils.napalm.call(
                DEVICE_CACHE,
                'get_facts',
                **{}
            )

    return GRAINS_CACHE

def _get_grain(name, proxy=None):
    '''
    Retrieves the grain value from the cached dictionary.
    '''
    grains = _retrieve_grains(proxy=proxy)
    if grains.get('result', False) and grains.get('out', {}):
        return grains.get('out').get(name)

def _get_device_grain(name):
    '''
    Retrieves device-specific grains.
    '''
    if not DEVICE_CACHE:
        return
    return DEVICE_CACHE.get(name.upper())

# ----------------------------------------------------------------------------------------------------------------------
# actual grains
# ----------------------------------------------------------------------------------------------------------------------

def getos():
    '''
    Returns the Operating System name running on the network device.

    Example: junos, iosxr, eos, ios etc.

    CLI Example - select all network devices running JunOS:

    .. code-block:: bash

        salt -G 'os:junos' test.ping
    '''
    return {'os': _get_device_grain('driver_name')}

def version(proxy=None):
    '''
    Returns the OS version.

    Example: 13.3R6.5, 6.0.2 etc.

    CLI Example - select all network devices running JunOS 13.3R6.5 and return the model:

    .. code-block:: bash

        salt -G 'os:junos and version:13.3R6.5' grains.get model

    Output:

    .. code-block:: yaml

        edge01.bjm01:
            MX2000
        edge01.sjc01:
            MX960
        edge01.mrs01:
            MX480
        edge01.muc01:
            MX240
    '''
    return {'version': _get_grain('os_version', proxy=proxy)}

def model(proxy=None):
    '''
    Returns the network device chassis model.

    Example: MX480, ASR-9904-AC etc.

    CLI Example - select all Juniper MX480 routers and execute traceroute to 8.8.8.8:

    .. code-block:: bash

        salt -G 'model:MX480' net.traceroute 8.8.8.8
    '''
    return {'model': _get_grain('model', proxy=proxy)}

def serial(proxy=None):
    '''
    Returns the chassis serial number.

    Example: FOX1234W00F

    CLI Example - select all devices whose serial number begins with `FOX` and display the serial number value:

    .. code-block:: bash

        salt -G 'serial:FOX*' grains.get serial

    Output:

    .. code-block:: yaml

        edge01.icn01:
            FOXW00F001
        edge01.del01:
            FOXW00F002
        edge01.yyz01:
            FOXW00F003
        edge01.mrs01:
            FOXW00F004
    '''
    return {'serial': _get_grain('serial_number', proxy=proxy)}

def vendor(proxy=None):
    '''
    Returns the network device vendor.

    Example: juniper, cisco, arista etc.

    CLI Example - select all devices produced by Cisco and shutdown:

    .. code-block:: bash

        salt -G 'vendor:cisco' net.cli "shut"
    '''
    return {'vendor': _get_grain('vendor', proxy=proxy)}

def uptime(proxy=None):
    '''
    Returns the uptime in seconds.

    CLI Example - select all devices started/restarted within the last hour:

    .. code-block: bash

        salt -G 'uptime<3600' test.ping
    '''
    return {'uptime': _get_grain('uptime', proxy=proxy)}

def interfaces(proxy=None):
    '''
    Returns the complete interfaces list of the network device.

    Example: ['lc-0/0/0', 'pfe-0/0/0', 'xe-1/3/0', 'lo0', 'irb', 'demux0', 'fxp0']

    CLI Example - select all devices that have a certain interface, e.g.: xe-1/1/1:

    .. code-block:: bash

        salt -G 'interfaces:xe-1/1/1' test.ping

    Output:

    .. code-block:: yaml

        edge01.yyz01:
            True
        edge01.maa01:
            True
        edge01.syd01:
            True
        edge01.del01:
            True
        edge01.dus01:
            True
        edge01.kix01:
            True
    '''
    return {'interfaces': _get_grain('interface_list', proxy=proxy)}

def username(proxy=None):
    '''
    Return the username

    CLI Example - select all devices using `foobar` as username for connection:

    .. code-block:: bash

        salt -G 'username:foobar' test.ping

    Output:

    .. code-block::yaml

        device1:
            True
        device2:
            True
    '''
    if proxy and salt.utils.napalm.is_proxy():
        # only if proxy will override the username
        # otherwise will use the default Salt grains
        return {'username': _get_device_grain('username')}

def hostname(proxy=None):
    '''
    Return the hostname as configured on the device.

    CLI Example:

    .. code-block:: bash

        salt 'device*' grains.get hostname

    Output:

    .. code-block:: yaml

        device1:
            edge01.yyz01
        device2:
            edge01.bjm01
        device3:
            edge01.flw01
    '''
    return {'hostname': _get_grain('hostname', proxy=proxy)}

def optional_args(proxy=None):
    '''
    Return the connection optional args.

    CLI Example - select all devices connection via port 1234:

    .. code-block:: bash

        salt -G 'optional_args:port:1234' test.ping

    Output:

    .. code-block:: yaml

        device1:
            True
        device2:
            True
    '''
    return {'optional_args': _get_device_grain('optional_args')}
<commit_msg>Pass the __opts__ var<commit_after># -*- coding: utf-8 -*-
'''
NAPALM Grains
=============

:codeauthor: Mircea Ulinic <mircea@cloudflare.com>
:maturity:   new
:depends:    napalm
:platform:   unix

Dependencies
------------

- :mod:`NAPALM proxy module <salt.proxies.napalm>`

.. versionadded:: 2016.11.0
'''

from __future__ import absolute_import

import logging
log = logging.getLogger(__name__)

# Salt lib
import salt.utils
import salt.utils.napalm

# ----------------------------------------------------------------------------------------------------------------------
# grains properties
# ----------------------------------------------------------------------------------------------------------------------

__virtualname__ = 'napalm'
__proxyenabled__ = ['napalm']

# ----------------------------------------------------------------------------------------------------------------------
# global variables
# ----------------------------------------------------------------------------------------------------------------------

GRAINS_CACHE = {}
DEVICE_CACHE = {}

# ----------------------------------------------------------------------------------------------------------------------
# property functions
# ----------------------------------------------------------------------------------------------------------------------

def __virtual__():
    return __virtualname__

# ----------------------------------------------------------------------------------------------------------------------
# helpers
# ----------------------------------------------------------------------------------------------------------------------

def _retrieve_grains(proxy=None):
    '''
    Retrieves the grains from the network device if not cached already.
    '''

    global GRAINS_CACHE
    global DEVICE_CACHE

    if not GRAINS_CACHE:
        if proxy and salt.utils.napalm.is_proxy(__opts__):
            # if proxy var passed and is NAPALM-type proxy minion
            GRAINS_CACHE = proxy['napalm.get_grains']()
            if 'napalm.get_device' in proxy:
                DEVICE_CACHE = proxy['napalm.get_device']()
        elif not proxy and salt.utils.napalm.is_minion():
            # if proxy var not passed and is running in a straight minion
            DEVICE_CACHE = salt.utils.napalm.get_device(__opts__)
            GRAINS_CACHE = salt.utils.napalm.call(
                DEVICE_CACHE,
                'get_facts',
                **{}
            )

    return GRAINS_CACHE

def _get_grain(name, proxy=None):
    '''
    Retrieves the grain value from the cached dictionary.
    '''
    grains = _retrieve_grains(proxy=proxy)
    if grains.get('result', False) and grains.get('out', {}):
        return grains.get('out').get(name)

def _get_device_grain(name):
    '''
    Retrieves device-specific grains.
    '''
    if not DEVICE_CACHE:
        return
    return DEVICE_CACHE.get(name.upper())

# ----------------------------------------------------------------------------------------------------------------------
# actual grains
# ----------------------------------------------------------------------------------------------------------------------

def getos():
    '''
    Returns the Operating System name running on the network device.

    Example: junos, iosxr, eos, ios etc.

    CLI Example - select all network devices running JunOS:

    .. code-block:: bash

        salt -G 'os:junos' test.ping
    '''
    return {'os': _get_device_grain('driver_name')}

def version(proxy=None):
    '''
    Returns the OS version.

    Example: 13.3R6.5, 6.0.2 etc.

    CLI Example - select all network devices running JunOS 13.3R6.5 and return the model:

    .. code-block:: bash

        salt -G 'os:junos and version:13.3R6.5' grains.get model

    Output:

    .. code-block:: yaml

        edge01.bjm01:
            MX2000
        edge01.sjc01:
            MX960
        edge01.mrs01:
            MX480
        edge01.muc01:
            MX240
    '''
    return {'version': _get_grain('os_version', proxy=proxy)}

def model(proxy=None):
    '''
    Returns the network device chassis model.

    Example: MX480, ASR-9904-AC etc.

    CLI Example - select all Juniper MX480 routers and execute traceroute to 8.8.8.8:

    .. code-block:: bash

        salt -G 'model:MX480' net.traceroute 8.8.8.8
    '''
    return {'model': _get_grain('model', proxy=proxy)}

def serial(proxy=None):
    '''
    Returns the chassis serial number.

    Example: FOX1234W00F

    CLI Example - select all devices whose serial number begins with `FOX` and display the serial number value:

    .. code-block:: bash

        salt -G 'serial:FOX*' grains.get serial

    Output:

    .. code-block:: yaml

        edge01.icn01:
            FOXW00F001
        edge01.del01:
            FOXW00F002
        edge01.yyz01:
            FOXW00F003
        edge01.mrs01:
            FOXW00F004
    '''
    return {'serial': _get_grain('serial_number', proxy=proxy)}

def vendor(proxy=None):
    '''
    Returns the network device vendor.

    Example: juniper, cisco, arista etc.

    CLI Example - select all devices produced by Cisco and shutdown:

    .. code-block:: bash

        salt -G 'vendor:cisco' net.cli "shut"
    '''
    return {'vendor': _get_grain('vendor', proxy=proxy)}

def uptime(proxy=None):
    '''
    Returns the uptime in seconds.

    CLI Example - select all devices started/restarted within the last hour:

    .. code-block: bash

        salt -G 'uptime<3600' test.ping
    '''
    return {'uptime': _get_grain('uptime', proxy=proxy)}

def interfaces(proxy=None):
    '''
    Returns the complete interfaces list of the network device.

    Example: ['lc-0/0/0', 'pfe-0/0/0', 'xe-1/3/0', 'lo0', 'irb', 'demux0', 'fxp0']

    CLI Example - select all devices that have a certain interface, e.g.: xe-1/1/1:

    .. code-block:: bash

        salt -G 'interfaces:xe-1/1/1' test.ping

    Output:

    .. code-block:: yaml

        edge01.yyz01:
            True
        edge01.maa01:
            True
        edge01.syd01:
            True
        edge01.del01:
            True
        edge01.dus01:
            True
        edge01.kix01:
            True
    '''
    return {'interfaces': _get_grain('interface_list', proxy=proxy)}

def username(proxy=None):
    '''
    Return the username

    CLI Example - select all devices using `foobar` as username for connection:

    .. code-block:: bash

        salt -G 'username:foobar' test.ping

    Output:

    .. code-block::yaml

        device1:
            True
        device2:
            True
    '''
    if proxy and salt.utils.napalm.is_proxy(__opts__):
        # only if proxy will override the username
        # otherwise will use the default Salt grains
        return {'username': _get_device_grain('username')}

def hostname(proxy=None):
    '''
    Return the hostname as configured on the device.

    CLI Example:

    .. code-block:: bash

        salt 'device*' grains.get hostname

    Output:

    .. code-block:: yaml

        device1:
            edge01.yyz01
        device2:
            edge01.bjm01
        device3:
            edge01.flw01
    '''
    return {'hostname': _get_grain('hostname', proxy=proxy)}

def optional_args(proxy=None):
    '''
    Return the connection optional args.

    CLI Example - select all devices connection via port 1234:

    .. code-block:: bash

        salt -G 'optional_args:port:1234' test.ping

    Output:

    .. code-block:: yaml

        device1:
            True
        device2:
            True
    '''
    return {'optional_args': _get_device_grain('optional_args')}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
MIT License

Copyright (c) 2016 Aarón Abraham Velasco Alvarez

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

"""
TODO Insertar funciones
"""
import php_grammar

class Inserter(object):
    def __init__(self, grammar, verbose):
        self.grammar = grammar
        self.verbose = verbose
        self.pattern_id = self.grammar.get_patterns()
        self.nonterminal_id = 0
        self.current_rule = 0
        self.index = 0
        self.inserting = False
        self.stack = []
        self.checked = []
        self.instructions = 0
        self.terminated = False
        self.pattern = 0
        self.trace = []

    def go_back(self, token):
        if self.nonterminal_id != 0:
            last = self.stack.pop()
            self.nonterminal_id = last[0]
            self.index = last[1] + 1
            self.checked = last[2]
            self.current_rule = last[5]
            self.insert(token)

    def insert_token(self, token, num_rule, total_rules):
        inserted, nonterminal = self.grammar.grammar[self.nonterminal_id].insert_token(self.current_rule,
                                                                                self.index,
                                                                                token,
                                                                                self.grammar.get_indices(),
                                                                                self.inserting,
                                                                                self.pattern_id)
        self.grammar.add_nonterminal(nonterminal)
        if not self.nonterminal_id == nonterminal.id:
            self.stack.append((self.nonterminal_id,
                               self.index,
                               self.checked,
                               num_rule,
                               total_rules,
                               self.current_rule))
            self.nonterminal_id = nonterminal.id
        if inserted == 1 or inserted == 3:
            self.checked = []
            self.index = 0
        self.insert(token)

    def insert(self, token):
        queue = self.grammar.grammar[self.nonterminal_id].get_expected_symbols_(self.current_rule,
                                                                                self.index,
                                                                                self.checked)
        symbol, num_rule, total_rules = None, None, None
        for expected in queue:
            self.current_rule, symbol, num_rule, total_rules = expected[0], expected[1], expected[2], expected[3]
            if self.verbose >= 3:
                print "debug:", self.nonterminal_id, self.current_rule, self.index, symbol, self.inserting
            if symbol.is_terminal():
                if symbol.token == token:
                    if token == php_grammar.separator or token in php_grammar.closing_tags:
                        self.terminated = True
                    if not self.terminated:
                        self.pattern = symbol.pattern_id
                        self.grammar.grammar[self.nonterminal_id].set_trace(self.current_rule,
                                                                            self.index,
                                                                            self.trace)
                        if (not token in php_grammar.opening_tags):
                            self.trace.append(token)
                    self.index += 1
                    break
                else:
                    self.checked.append(self.current_rule)
                    self.inserting = False
            elif symbol.is_nonterminal():
                self.stack.append((self.nonterminal_id,
                                   self.index,
                                   self.checked,
                                   num_rule,
                                   total_rules,
                                   self.current_rule))
                self.nonterminal_id = symbol.id
                self.index = 0
                self.checked = []
                self.inserting = False
                self.insert(token)
                break
            else:
                if not self.inserting:
                    self.go_back(token)
                else:
                    if not (token == php_grammar.separator or token in php_grammar.closing_tags):
                        self.insert_token(token, num_rule, total_rules)
                        self.inserting = True
                    else:
                        self.terminated = True
                        self.inserting = False
                        self.go_back(token)
                break
        else:
            if self.nonterminal_id != 0:
                last = self.stack.pop()
                if last[3] < last[4] - 1 and self.index == 0:
                    self.nonterminal_id = last[0]
                    self.index = 0
                    self.checked = last[2]
                    self.checked.append(last[5])
                    self.insert(token)
                else:
                    if not (token == php_grammar.separator or token in php_grammar.closing_tags):
                        self.stack.append(last)
                        self.insert_token(token, num_rule, total_rules)
                        self.inserting = True
                    else:
                        self.terminated = True
                        self.inserting = False
                        self.nonterminal_id = last[0]
                        self.index = last[1] + 1
                        self.checked = last[2]
                        self.current_rule = last[5]
                        self.insert_token(0, num_rule, total_rules)
            else:
                self.insert_token(token, num_rule, total_rules)
                self.inserting = True

    def run(self, tokens):
        is_php = False
        for item in tokens:
            token, lexeme = item[0], item[1]
            if token in php_grammar.opening_tags:
                is_php = True
            elif token in php_grammar.closing_tags:
                is_php = False

            if is_php or token in php_grammar.closing_tags:
                if self.verbose >= 3:
                    print "========== %s %s ==========" % (token, lexeme)
                self.insert(token)
                if self.terminated:
                    if self.pattern == self.pattern_id:
                        self.instructions += 1
                    self.terminated = False
                    if self.verbose >= 3:
                        print "Pattern:", self.pattern, len(self.trace), self.trace
                    self.pattern = 0
                    self.trace = []
<commit_msg>Added files via upload<commit_after># -*- coding: utf-8 -*-

"""
MIT License

Copyright (c) 2016 Aarón Abraham Velasco Alvarez

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

#TODO Insertar funciones

import php_grammar

class Inserter(object):
    def __init__(self, grammar, verbose):
        self.grammar = grammar
        self.verbose = verbose
        self.pattern_id = self.grammar.get_patterns()
        self.nonterminal_id = 0
        self.current_rule = 0
        self.index = 0
        self.inserting = False
        self.stack = []
        self.checked = []
        self.instructions = 0
        self.terminated = False
        self.pattern = 0
        self.trace = []

    def go_back(self, token):
        if self.nonterminal_id != 0:
            last = self.stack.pop()
            self.nonterminal_id = last[0]
            self.index = last[1] + 1
            self.checked = last[2]
            self.current_rule = last[5]
            self.insert(token)

    def insert_token(self, token, num_rule, total_rules):
        inserted, nonterminal = self.grammar.grammar[self.nonterminal_id].insert_token(self.current_rule,
                                                                                self.index,
                                                                                token,
                                                                                self.grammar.get_indices(),
                                                                                self.inserting,
                                                                                self.pattern_id)
        self.grammar.add_nonterminal(nonterminal)
        if not self.nonterminal_id == nonterminal.id:
            self.stack.append((self.nonterminal_id,
                               self.index,
                               self.checked,
                               num_rule,
                               total_rules,
                               self.current_rule))
            self.nonterminal_id = nonterminal.id
        if inserted == 1 or inserted == 3:
            self.checked = []
            self.index = 0
        self.insert(token)

    def insert(self, token):
        queue = self.grammar.grammar[self.nonterminal_id].get_expected_symbols_(self.current_rule,
                                                                                self.index,
                                                                                self.checked)
        symbol, num_rule, total_rules = None, None, None
        for expected in queue:
            self.current_rule, symbol, num_rule, total_rules = expected[0], expected[1], expected[2], expected[3]
            if self.verbose >= 3:
                print "debug:", self.nonterminal_id, self.current_rule, self.index, symbol, self.inserting
            if symbol.is_terminal():
                if symbol.token == token:
                    if token == php_grammar.separator or token in php_grammar.closing_tags:
                        self.terminated = True
                    if not self.terminated:
                        self.pattern = symbol.pattern_id
                        self.grammar.grammar[self.nonterminal_id].set_trace(self.current_rule,
                                                                            self.index,
                                                                            self.trace)
                        if (not token in php_grammar.opening_tags):
                            self.trace.append(token)
                    self.index += 1
                    break
                else:
                    self.checked.append(self.current_rule)
                    self.inserting = False
            elif symbol.is_nonterminal():
                self.stack.append((self.nonterminal_id,
                                   self.index,
                                   self.checked,
                                   num_rule,
                                   total_rules,
                                   self.current_rule))
                self.nonterminal_id = symbol.id
                self.index = 0
                self.checked = []
                self.inserting = False
                self.insert(token)
                break
            else:
                if not self.inserting:
                    self.go_back(token)
                else:
                    if not (token == php_grammar.separator or token in php_grammar.closing_tags):
                        self.insert_token(token, num_rule, total_rules)
                        self.inserting = True
                    else:
                        self.terminated = True
                        self.inserting = False
                        self.go_back(token)
                break
        else:
            if self.nonterminal_id != 0:
                last = self.stack.pop()
                if last[3] < last[4] - 1 and self.index == 0:
                    self.nonterminal_id = last[0]
                    self.index = 0
                    self.checked = last[2]
                    self.checked.append(last[5])
                    self.insert(token)
                else:
                    if not (token == php_grammar.separator or token in php_grammar.closing_tags):
                        self.stack.append(last)
                        self.insert_token(token, num_rule, total_rules)
                        self.inserting = True
                    else:
                        self.terminated = True
                        self.inserting = False
                        self.nonterminal_id = last[0]
                        self.index = last[1] + 1
                        self.checked = last[2]
                        self.current_rule = last[5]
                        self.insert_token(0, num_rule, total_rules)
            else:
                self.insert_token(token, num_rule, total_rules)
                self.inserting = True

    def run(self, tokens):
        is_php = False
        for item in tokens:
            token, lexeme = item[0], item[1]
            if token in php_grammar.opening_tags:
                is_php = True
            elif token in php_grammar.closing_tags:
                is_php = False

            if is_php or token in php_grammar.closing_tags and not token in php_grammar.ignored_tokens:
                if self.verbose >= 3:
                    print "========== %s %s ==========" % (token, lexeme)
                self.insert(token)
                if self.terminated:
                    if self.pattern == self.pattern_id:
                        self.instructions += 1
                    self.terminated = False
                    if self.verbose >= 3:
                        print "Pattern:", self.pattern, len(self.trace), self.trace
                    self.pattern = 0
                    self.trace = []
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This package provides support for parsingNetCDF file headers of format
version 1, also known as the 'classic' format. This format was used
exclusively up to NetCDF library version 3.5

The class NC3File is used to parse and represent header information.
There is no direct support as yet for reading the actual data.
"""

import os, os.path, re, struct, hashlib

from logger import Logger, LOGGER_TRACE, LOGGER_INFO, LOGGER_WARNING

# -- tags found in NetCDF files --
NC_BYTE      =  1
NC_CHAR      =  2
NC_SHORT     =  3
NC_LONG      =  4
NC_FLOAT     =  5
NC_DOUBLE    =  6

NC_DIMENSION = 10
NC_VARIABLE  = 11
NC_ATTRIBUTE = 12

class NC3Type:
    """
    Represents a NetCDF data type. Accessible fields:

    nc_code - the integral tag specifying the type in NetCDF headers
    name    - the human readable type name
    size    - the size in bytes for an element of this type
    py_code - the corresponding python type code for the struct package
    """
    def __init__(self, nc_code, name, size, py_code):
        self.nc_code = nc_code
        self.name = name
        self.size = size
        self.py_code = py_code

# -- fill the global list of available data types --
NC_TYPE = {}
for (nc_code, name, size, py_code) in (
  (NC_BYTE,   "byte",   1, "b"),
  (NC_CHAR,   "char",   1, "c"),
  (NC_SHORT,  "short",  2, "h"),
  (NC_LONG,   "int",    4, "i"),
  (NC_FLOAT,  "float",  4, "f"),
  (NC_DOUBLE, "double", 8, "d")
  ):
    NC_TYPE[nc_code] = NC3Type(nc_code, name, size, py_code)

def fmt(val):
    "Recursively formats an attribute value."
    if type(val) is str:
        # -- escape all double quotes that aren't already escaped
        result = []
        escaped = False
        for piece in re.split('(")', val):
            if piece == '"' and not escaped:
                result.append('\\')
            result.append(piece)
            escaped = piece.endswith('\\')
        tmp = ''.join(result)
        # -- split along line breaks and quote each line separately
        return ('"%s"' % tmp).replace('\n', '\\n",\n\t\t\t"')
    elif type(val) in ( tuple, list ):
        # -- format aggregates as comma-separated lists
        return ", ".join(map(fmt, val))
    else:
        # -- everything else should be simply a number
        return str(val)

class NC3Dimension:
    """
    Represents a dimension. Accessible fields are 'name' and 'value'.
    """
    def __init__(self, name, value):
        self.name = name
        self.value = value

    def __str__(self):
        return "%s = %d" % (self.name, self.value)

class NC3Attribute:
    """
    Represents an attribute. Accessible fields are 'name' and 'value'.
    """
    def __init__(self, name, value):
        self.name = name
        self.value = value

    def __str__(self):
        return "%s = %s" % (self.name, fmt(self.value))

class NC3Variable:
    """
    Represents a variable. Useful properties:

    name             - the name of the variable
    dimensions       - the list of dimensions (class NC3Dimension)
    attributes       - the list of attributes (class NC3Attribute)
    data_size        - the size of the associated data in bytes
    data_start       - the data offset in bytes within the file
    type_name        - the human readable name of the data type
    python_type_code - the type code to use with the struct package
    element_size     - the size of a data element
    """
    def __init__(self, name, dimensions, attributes,
                 nc_type, data_size, data_start):
        self.name = name
        self.dimensions = dimensions
        self.attributes = attributes
        self.nc_type = nc_type
        self.data_size = data_size
        self.data_start = data_start

    @property
    def type_name(self):
        return NC_TYPE[self.nc_type].name

    @property
    def python_type_code(self):
        return NC_TYPE[self.nc_type].py_code

    @property
    def element_size(self):
        return NC_TYPE[self.nc_type].size

    def __str__(self):
        return "%s %s(%s)" % (self.type_name, self.name,
                              ", ".join((d.name for d in self.dimensions)))

def read_values(fp, type_code, number):
    tp = NC_TYPE[type_code]
    size = tp.size * number
    value = fp.read(size)
    if len(value) < size:
        raise RuntimeError("Premature end of file.")
    fp.read(3 - (size + 3) % 4)
    if type_code == NC_CHAR:
        return value
    else:
        return struct.unpack(">%d%s" % (number, tp.py_code), value)

def read_integer(fp):
    return read_values(fp, NC_LONG, 1)[0]

def read_non_negative(fp):
    n = read_integer(fp)
    if n < 0:
        raise RuntimeError("Non-negative number expected")
    return n

def read_string(fp):
    size = read_non_negative(fp)
    return read_values(fp, NC_CHAR, size)

def read_dimensions(fp):
    dimensions = []
    tag = read_integer(fp)
    ndims = read_non_negative(fp)
    if tag == NC_DIMENSION:
        for i in range(ndims):
            name = read_string(fp)
            size = read_non_negative(fp)
            dimensions.append(NC3Dimension(name, size))
    elif tag != 0 or ndims != 0:
        raise RuntimeError("Expected dimension array.")

    return dimensions

def read_attributes(fp):
    attributes = []
    tag = read_integer(fp)
    nattr = read_non_negative(fp)
    if tag == NC_ATTRIBUTE:
        for i in range(nattr):
            name = read_string(fp)
            type = read_integer(fp)
            size = read_non_negative(fp)
            values = read_values(fp, type, size)
            attributes.append(NC3Attribute(name, values))
    elif tag != 0 or nattr != 0:
        raise RuntimeError("Expected attribute array.")

    return attributes

def read_variables(fp, dimensions):
    variables = []
    tag = read_integer(fp)
    nvars = read_non_negative(fp)
    if tag == NC_VARIABLE:
        for i in range(nvars):
            name = read_string(fp)
            ndims = read_non_negative(fp)
            dims = tuple(dimensions[read_non_negative(fp)]
                         for x in range(ndims))
            attr = read_attributes(fp)
            nc_type = read_integer(fp)
            size = read_non_negative(fp)
            start = read_non_negative(fp)
            variables.append(NC3Variable(name, dims, attr, nc_type, size, start))
    elif tag != 0 or nvars != 0:
        raise RuntimeError("Expected variable descriptions.")

    return variables

class MD5Wrapper:
    def __init__(self, fp):
        self._fp = fp
        self._count = 0
        self._md5 = hashlib.md5()

    def read(self, size):
        data = self._fp.read(size)
        self._count += len(data)
        self._md5.update(data)
        return data

    def count(self):
        return self._count

    def hexdigest(self):
        return self._md5.hexdigest()

class NC3File:
    """
    Represents the complete header data from a NetCDF file. The constructor
    accepts any object with a read() method for parsing that data.

    Useful properties:

    dimensions  - the list of dimensions defined (type NC3Dimension)
    attributes  - the list of attributes defined (type NC3Attribute)
    variables   - the list of variables (type NC3Variable)
    header_size - the header size on file in bytes
    fingerprint - the MD5 hexdigest value of the header contents
    """
    def __init__(self, fp):
        # -- some logging
        log = Logger()
        log.writeln("Parsing NetCDF header")
        log.enter()

        # -- wrap the input stream and parse from it
        fp = MD5Wrapper(fp)

        magic = read_values(fp, NC_CHAR, 4)
        if magic != "CDF\001":
            raise RuntimeError("Not a NetCDF version 1 file.")

        self.numrecords = read_non_negative(fp)
        self.dimensions = read_dimensions(fp)
        self.attributes = read_attributes(fp)
        self.variables  = read_variables(fp, self.dimensions)

        self.header_size = fp.count()
        self.fingerprint = fp.hexdigest()

        # -- more logging
        log.trace("Header size is %d." % self.header_size)
        log.leave()

def looksLikeNetCDF(name):
    if name.endswith(".bz2"):
        name = name[:-4]
    return name.endswith('_nc') or name.endswith('.nc')

def basenameNetCDF(name):
    if name.endswith(".bz2"):
        name = name[:-4]
    if name.endswith(".nc") or name.endswith("_nc"):
        name = name[:-3]
    return name

def nc3file_from_directory(path):
    from file_cache import FileCache

    # --- normalize the path name
    if path.endswith('/'):
        path = path[:-1]

    # -- determine the base name for the header file
    name = basenameNetCDF(os.path.basename(path))

    # -- collect the files under the given path
    if os.path.isdir(path):
        entries = list(os.path.join(root, f)
                       for (root, dirs, files) in os.walk(path)
                       for f in files
                       if looksLikeNetCDF(f))
        entries.sort()
    else:
        entries = [ path ]
    if not entries:
        raise RuntimeError("%s: no NetCDF files found." % path)

    # -- open the first file and return the object
    fp = FileCache(entries[0])

    try:
        info = NC3File(fp)
    finally:
        fp.close()

    return info

if __name__ == "__main__":
    import sys

    i = 1
    if sys.argv[i].startswith("-v"):
        if sys.argv[i].startswith("-vv"):
            Logger().priority = LOGGER_TRACE
        else:
            Logger().priority = LOGGER_INFO
        i += 1

    nc3file = nc3file_from_directory(sys.argv[i])

    buffer = [ "netcdf %s {" % nc3file.name ]

    buffer.append("dimensions:")
    for dim in nc3file.dimensions:
        buffer.append("\t%s = %s ;" % (dim.name, dim.value))

    buffer.append("variables:")
    for var in nc3file.variables:
        buffer.append("\t%s ;" % var)
        for attr in var.attributes:
            buffer.append("\t\t%s:%s ;" % (var.name, attr))

    buffer.append("")
    buffer.append("// global attributes:")
    for attr in nc3file.attributes:
        buffer.append("\t\t:%s ;" % attr)

    buffer.append("}")

    fp = file(("%s.cdf" % nc3file.name), 'wb')
    fp.write('\n'.join(buffer) + '\n')
    fp.close()
<commit_msg>removed logging and fixed main code in nc3header.py<commit_after>#!/usr/bin/env python

"""
This package provides support for parsingNetCDF file headers of format
version 1, also known as the 'classic' format. This format was used
exclusively up to NetCDF library version 3.5

The class NC3File is used to parse and represent header information.
There is no direct support as yet for reading the actual data.
"""

import os, os.path, re, struct, hashlib

# -- tags found in NetCDF files --
NC_BYTE      =  1
NC_CHAR      =  2
NC_SHORT     =  3
NC_LONG      =  4
NC_FLOAT     =  5
NC_DOUBLE    =  6

NC_DIMENSION = 10
NC_VARIABLE  = 11
NC_ATTRIBUTE = 12

class NC3Type:
    """
    Represents a NetCDF data type. Accessible fields:

    nc_code - the integral tag specifying the type in NetCDF headers
    name    - the human readable type name
    size    - the size in bytes for an element of this type
    py_code - the corresponding python type code for the struct package
    """
    def __init__(self, nc_code, name, size, py_code):
        self.nc_code = nc_code
        self.name = name
        self.size = size
        self.py_code = py_code

# -- fill the global list of available data types --
NC_TYPE = {}
for (nc_code, name, size, py_code) in (
  (NC_BYTE,   "byte",   1, "b"),
  (NC_CHAR,   "char",   1, "c"),
  (NC_SHORT,  "short",  2, "h"),
  (NC_LONG,   "int",    4, "i"),
  (NC_FLOAT,  "float",  4, "f"),
  (NC_DOUBLE, "double", 8, "d")
  ):
    NC_TYPE[nc_code] = NC3Type(nc_code, name, size, py_code)

def fmt(val):
    "Recursively formats an attribute value."
    if type(val) is str:
        # -- escape all double quotes that aren't already escaped
        result = []
        escaped = False
        for piece in re.split('(")', val):
            if piece == '"' and not escaped:
                result.append('\\')
            result.append(piece)
            escaped = piece.endswith('\\')
        tmp = ''.join(result)
        # -- split along line breaks and quote each line separately
        return ('"%s"' % tmp).replace('\n', '\\n",\n\t\t\t"')
    elif type(val) in ( tuple, list ):
        # -- format aggregates as comma-separated lists
        return ", ".join(map(fmt, val))
    else:
        # -- everything else should be simply a number
        return str(val)

class NC3Dimension:
    """
    Represents a dimension. Accessible fields are 'name' and 'value'.
    """
    def __init__(self, name, value):
        self.name = name
        self.value = value

    def __str__(self):
        return "%s = %d" % (self.name, self.value)

class NC3Attribute:
    """
    Represents an attribute. Accessible fields are 'name' and 'value'.
    """
    def __init__(self, name, value):
        self.name = name
        self.value = value

    def __str__(self):
        return "%s = %s" % (self.name, fmt(self.value))

class NC3Variable:
    """
    Represents a variable. Useful properties:

    name             - the name of the variable
    dimensions       - the list of dimensions (class NC3Dimension)
    attributes       - the list of attributes (class NC3Attribute)
    data_size        - the size of the associated data in bytes
    data_start       - the data offset in bytes within the file
    type_name        - the human readable name of the data type
    python_type_code - the type code to use with the struct package
    element_size     - the size of a data element
    """
    def __init__(self, name, dimensions, attributes,
                 nc_type, data_size, data_start):
        self.name = name
        self.dimensions = dimensions
        self.attributes = attributes
        self.nc_type = nc_type
        self.data_size = data_size
        self.data_start = data_start

    @property
    def type_name(self):
        return NC_TYPE[self.nc_type].name

    @property
    def python_type_code(self):
        return NC_TYPE[self.nc_type].py_code

    @property
    def element_size(self):
        return NC_TYPE[self.nc_type].size

    def __str__(self):
        return "%s %s(%s)" % (self.type_name, self.name,
                              ", ".join((d.name for d in self.dimensions)))

def read_values(fp, type_code, number):
    tp = NC_TYPE[type_code]
    size = tp.size * number
    value = fp.read(size)
    if len(value) < size:
        raise RuntimeError("Premature end of file.")
    fp.read(3 - (size + 3) % 4)
    if type_code == NC_CHAR:
        return value
    else:
        return struct.unpack(">%d%s" % (number, tp.py_code), value)

def read_integer(fp):
    return read_values(fp, NC_LONG, 1)[0]

def read_non_negative(fp):
    n = read_integer(fp)
    if n < 0:
        raise RuntimeError("Non-negative number expected")
    return n

def read_string(fp):
    size = read_non_negative(fp)
    return read_values(fp, NC_CHAR, size)

def read_dimensions(fp):
    dimensions = []
    tag = read_integer(fp)
    ndims = read_non_negative(fp)
    if tag == NC_DIMENSION:
        for i in range(ndims):
            name = read_string(fp)
            size = read_non_negative(fp)
            dimensions.append(NC3Dimension(name, size))
    elif tag != 0 or ndims != 0:
        raise RuntimeError("Expected dimension array.")

    return dimensions

def read_attributes(fp):
    attributes = []
    tag = read_integer(fp)
    nattr = read_non_negative(fp)
    if tag == NC_ATTRIBUTE:
        for i in range(nattr):
            name = read_string(fp)
            type = read_integer(fp)
            size = read_non_negative(fp)
            values = read_values(fp, type, size)
            attributes.append(NC3Attribute(name, values))
    elif tag != 0 or nattr != 0:
        raise RuntimeError("Expected attribute array.")

    return attributes

def read_variables(fp, dimensions):
    variables = []
    tag = read_integer(fp)
    nvars = read_non_negative(fp)
    if tag == NC_VARIABLE:
        for i in range(nvars):
            name = read_string(fp)
            ndims = read_non_negative(fp)
            dims = tuple(dimensions[read_non_negative(fp)]
                         for x in range(ndims))
            attr = read_attributes(fp)
            nc_type = read_integer(fp)
            size = read_non_negative(fp)
            start = read_non_negative(fp)
            variables.append(NC3Variable(name, dims, attr, nc_type, size, start))
    elif tag != 0 or nvars != 0:
        raise RuntimeError("Expected variable descriptions.")

    return variables

class MD5Wrapper:
    def __init__(self, fp):
        self._fp = fp
        self._count = 0
        self._md5 = hashlib.md5()

    def read(self, size):
        data = self._fp.read(size)
        self._count += len(data)
        self._md5.update(data)
        return data

    def count(self):
        return self._count

    def hexdigest(self):
        return self._md5.hexdigest()

class NC3File:
    """
    Represents the complete header data from a NetCDF file. The constructor
    accepts any object with a read() method for parsing that data.

    Useful properties:

    dimensions  - the list of dimensions defined (type NC3Dimension)
    attributes  - the list of attributes defined (type NC3Attribute)
    variables   - the list of variables (type NC3Variable)
    header_size - the header size on file in bytes
    fingerprint - the MD5 hexdigest value of the header contents
    """
    def __init__(self, fp):
        fp = MD5Wrapper(fp)

        magic = read_values(fp, NC_CHAR, 4)
        if magic != "CDF\001":
            raise RuntimeError("Not a NetCDF version 1 file.")

        self.numrecords = read_non_negative(fp)
        self.dimensions = read_dimensions(fp)
        self.attributes = read_attributes(fp)
        self.variables  = read_variables(fp, self.dimensions)

        self.header_size = fp.count()
        self.fingerprint = fp.hexdigest()

def looksLikeNetCDF(name):
    if name.endswith(".bz2"):
        name = name[:-4]
    return name.endswith('_nc') or name.endswith('.nc')

def basenameNetCDF(name):
    if name.endswith(".bz2"):
        name = name[:-4]
    if name.endswith(".nc") or name.endswith("_nc"):
        name = name[:-3]
    return name

def nc3file_from_directory(path):
    from file_cache import FileCache

    # --- normalize the path name
    if path.endswith('/'):
        path = path[:-1]

    # -- determine the base name for the header file
    name = basenameNetCDF(os.path.basename(path))

    # -- collect the files under the given path
    if os.path.isdir(path):
        entries = list(os.path.join(root, f)
                       for (root, dirs, files) in os.walk(path)
                       for f in files
                       if looksLikeNetCDF(f))
        entries.sort()
    else:
        entries = [ path ]
    if not entries:
        raise RuntimeError("%s: no NetCDF files found." % path)

    # -- open the first file and return the object
    fp = FileCache(entries[0])

    try:
        info = NC3File(fp)
    finally:
        fp.close()

    return info

if __name__ == "__main__":
    import sys

    path = sys.argv[1]
    name = os.path.splitext(os.path.basename(path))[0]
    nc3file = nc3file_from_directory(sys.argv[1])

    buffer = [ "netcdf %s {" % name ]

    buffer.append("dimensions:")
    for dim in nc3file.dimensions:
        buffer.append("\t%s = %s ;" % (dim.name, dim.value))

    buffer.append("variables:")
    for var in nc3file.variables:
        buffer.append("\t%s ;" % var)
        for attr in var.attributes:
            buffer.append("\t\t%s:%s ;" % (var.name, attr))

    buffer.append("")
    buffer.append("// global attributes:")
    for attr in nc3file.attributes:
        buffer.append("\t\t:%s ;" % attr)

    buffer.append("}")

    fp = file(("%s.cdf" % name), 'wb')
    fp.write('\n'.join(buffer) + '\n')
    fp.close()
<|endoftext|>