<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------------------------------------------
#  libTML:  A BEEP based Messaging Suite
#  Copyright (C) 2015 wobe-systems GmbH
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public License
#  as published by the Free Software Foundation; either version 2.1
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this program; if not, write to the Free
#  Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
#  02111-1307 USA
#
#  You may find a copy of the license under this software is released
#  at COPYING file. This is LGPL software: you are welcome to develop
#  proprietary applications using this library without any royalty or
#  fee but returning back any change, improvement or addition in the
#  form of source code, project image, documentation patches, etc.
#
#  Homepage:
#    http://www.libtml.org
#
#  For professional support contact us:
#
#    wobe-systems GmbH
#    support@libtml.org
#-----------------------------------------------------------

import sidex
import sidex.variant

from sidex.constants import *

## @defgroup SDXDocGroup SIDEX document
#  @ingroup highLevelClasses
#  @brief Using SIDEX XML documents.
#
#  The SIDEX document manages a group/key based data container. Data types are
#  handled automatically by the document class and the variant data objects to ensure the type is
#  saved together with the data.
#
#  @section SIDEX Document example
#  The example shows the creation of a sidex document and how to add some basic data in the \c General group.
#  The \c Container group contains a list of basic types and other container types like a list
#  and a dictionary. Nesting of container types is the foundation to handle complex data structures.
#
#  @snippet sdx_doc_01.py basic_usage
#
#  The resulting SIDEX XML file contains all data types.
#
#  @include sdx_doc_01_result.xml

## @brief SIDEX parameter group.
#  @ingroup SDXDocGroup
#
#  The class emulates a dictionary to simplify access of the key value pairs
#  in a parameter group.
class SDXDocumentGroup(object):

    ## @brief Constructor
    #
    #  @param sdxdoc  :[\b SDXDocument] parent document
    #  @param grpname :[\b str] name of the group
    def __init__(self, sdxdoc, grpname):
        self.__grpname = sidex.variant.sdx_to_py(grpname)
        self.__sdxdoc = sdxdoc

    ## @brief Get an value by key name.
    #
    #  To read a value from a SIDEX document the SDXDocument instance can be used like
    #  a nested dictionary.
    #
    #  @code{.py}
    #  value = sdx_document['General']['ParamKey']
    #  @endcode
    #
    #  @param key : [\b str] name of the key
    #
    #  @returns [\b sidex.variant.SDXBase] the value
    #
    #  @exception KeyError the requested key does not exist
    #
    #  @sa SDXDcoument.__getitem__(), SDXDocument.read_value(), SDXDocument.autocast
    def __getitem__(self, key):
        lkey = sidex.variant.sdx_to_py(key)
        if sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, lkey):
            if self.__sdxdoc.autocast:
                return sidex.variant.sdx_to_py(self.__sdxdoc.read_value(self.__grpname, lkey))
            else:
                return self.__sdxdoc.read_value(self.__grpname, lkey)
        else:
            raise KeyError("invalid key %s" % lkey)

    ## @brief Add or replace a key value pair.
    #
    #  Values can be added to a document by assigning to the double indexed SDXDocument instance.
    #
    #  @code
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  sdoc['MyGroup']['MyKey'] = 3.14
    #  @endcode
    #
    #  @param key   : [\b str] name of the key
    #  @param value : [\b sidex.variant.SDXBase, \b str, \b list, \b tuple, \b dict, \b float,
    #    \b int, \b bool, \b none] value to write
    #
    #  @sa SDXDocument.write_value()
    def __setitem__(self, key, value):
        self.__sdxdoc.write_value(self.__grpname, sidex.variant.sdx_to_py(key), value)

    ## @brief Delete a key value pair from the document.
    #
    #  To Deleta a key/value pair from the document the \c del command
    #  can be used.
    #
    #  @code{.py}
    #  del sdx_document['MyGroup']['MyParamKey']
    #  @endcode
    #
    #  @param key : [\b str] name of the key
    def __delitem__(self, key):
        lkey = sidex.variant.sdx_to_py(key)
        if sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, lkey):
            sidex.sidex_DeleteKey(self.__sdxdoc.shandle, self.__grpname, lkey)

    ## @brief Get a list of key names.
    #
    #  @code{.py}
    #  keynames = sdxdocument['MyGroup'].keys()
    #  @endcode
    #
    #  @returns [\b sidex.variant.SDXList] a list of strings containing the keys
    #
    #  @sa SDXDocumentGroup.keys()
    def keys(self):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_GetKeys(self.__sdxdoc.shandle, self.__grpname))

    ## @brief Get the number of keys in a group.
    #
    #  @returns [\b int] number of keys
    def __len__(self):
        return len(self.keys())

    ## @brief Check if a key with name exists.
    #
    #  @param name : [\b str] key name
    #
    #  @returns [\b bool] True if the key exists
    def has_key(self, name):
        return sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, sidex.variant.sdx_to_py(name))

## @brief SIDEX document.
#  @ingroup SDXDocGroup
#
#  A SIDEX document can be converted to a SIDEX XML string to be saved as a file or to be send to as a message
#  to another process or host. If a SIDEX file or string has to be parsed, it is simply loaded into a SIDEX
#  document that enables type safe access and modification.
class SDXDocument(object):

    ## @brief Constructor
    #
    #  @param document  :[\b int, \b str] pass either the handle of a
    #    sidex document or the name of the sidex document. The name will be used as root node.
    def __init__(self, document=None, content=None):
        if document is None:
            document = "SDXRoot"

        if PY3 and isinstance(document, str):
            self.__thisown = True
            self.__shandle = sidex.sidex_Create(document)
        elif PY2 and isinstance(document, (str, unicode)):
            self.__thisown = True
            self.__shandle = sidex.sidex_Create(document)
        elif PY3 and isinstance(document, int):
            self.__thisown = False
            self.__shandle = document
        elif PY2 and isinstance(document, long):
            self.__thisown = False
            self.__shandle = document
        else:
            raise TypeError("invalid document type %s" % (type(document)))

        if content is not None:
            self.content = content

        self.__autocast = True

        return

        # ------------ properties -----------------
        # the code here is for documentation only
        self.shandle = None
        self.content = ''
        self.document_name = ''
        self.autocast = True
        # ------------ properties -----------------

    ## @brief Destructor of SDXDocument.
    #
    #  If the document object owns the SIDEX handle, it will be freed on destruction.
    def __del__(self):
        if self.__thisown:
            sidex.sidex_Free(self.__shandle)

    ## @property autocast
    #  @brief [\b bool] automatically cast SDX variant objects to python.
    #
    #  If values are read from the document with __getitem__, they are automatically
    #  casted to python objects. If this cast is not necessary assign False to this property.
    #
    #  @par Access:
    #    read/write
    #

    #  @cond NODOC
    @property
    def autocast(self):
        return self.__autocast

    @autocast.setter
    def autocast(self, value):
        self.__autocast = value
    #  @endcond

    ## @property document_name
    #  @brief [\b str] document and root node name in XML data
    #
    #  @par Access:
    #    read/write
    #
    #  @sa sidex_Get_DocumentName(), sidex_Set_DocumentName()

    #  @cond NODOC
    @property
    def document_name(self):
        return sidex.sidex_Get_DocumentName(self.__shandle)

    @document_name.setter
    def document_name(self, value):
        sidex.sidex_Set_DocumentName(self.__shandle, value)
    #  @endcond

    ## @property shandle
    #  @brief [\b None] SIDEX document handle
    #
    #  @par Access:
    #    read

    #  @cond NODOC
    @property
    def shandle(self):
        return self.__shandle
    #  @endcond

    ## @property content
    #  @brief [\b str] Accessing XML content of the document.
    #
    #  @par Access:
    #    read/write
    #
    #  @sa __str__(), sidex_Get_Content(), sidex_Set_Content()

    #  @cond NODOC
    @property
    def content(self):
        return sidex.sidex_Get_Content(self.__shandle)

    @content.setter
    def content(self, value):
        sidex.sidex_Set_Content(self.__shandle, sidex.variant.sdx_to_py(value))
    #  @endcond

    ## @brief Access a parameter group by name.
    #
    #  @param key : [\b str] name of the parameter group
    #
    #  @returns   [\b SDXDocumentGroup] parameter group object
    #
    #  @exception KeyError key does not exist
    def __getitem__(self, key):
        return SDXDocumentGroup(self, key)

    ## @brief Set a group of values.
    #
    #  Setting a group of values directly requires a dictionary with alphanumeric keys matching
    #  the specification for XML tag names.
    #
    #  @code{.py}
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  sdoc['MyGroup'] = {'KeyA':42, 'PI':3.14}
    #  @endcode
    #
    #  An existing group will be replaced.
    #
    #  @param key   : [\b str] name of the group
    #  @param value : [\b dict] dictionary with string keys and values
    #
    #  @exception TypeError group does not match the requirements
    def __setitem__(self, key, value):
        if not isinstance(value, dict):
            raise TypeError("value is not a dictionary")

        for lkey in value.keys():
            if not isinstance(key, str):
                raise TypeError("key in group in not a string %s" % lkey)

        if sidex.sidex_HasGroup(self.__shandle, key):
            self.__delitem__(key)

        for lkey, lvalue in value.items():
            self.write_value(key, lkey, lvalue)

    ## @brief Delete a group from the document.
    #
    #  To delete a complete parameter group from the document the \c del command
    #  can be used.
    #
    #  @code{.py}
    #  del sdx_document['MyGroup']
    #  @endcode
    #
    #  @param key : [\b str] name of the group
    def __delitem__(self, key):
        if sidex.sidex_HasGroup(self.__shandle, key):
            sidex.sidex_DeleteGroup(self.__shandle, key)

    ## @brief Get a list of group names.
    #
    #  The SIDEX document works more a less like a python dict. That's why the function name is \c keys
    #  and not \c groups. In fact it returns a list of group names.
    #
    #  @code{.py}
    #  groupnames = sdxdocument.keys()
    #  @endcode
    #
    #  @returns [\b sidex.variant.SDXList] a list of strings containing the keys
    #
    #  @sa SDXDocumentGroup.keys()
    def keys(self):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_GetGroups(self.__shandle), owner=True)

    ## @brief Get the number of groups in the document.
    #
    #  @returns [\b int] number of groups
    def __len__(self):
        return len(self.keys())

    ## @brief Convert the document data into a SIDEX XML string.
    #
    #  @returns [\b str] SIDEX formatted XML string
    def __str__(self):
        return self.content

    ## @brief Write a value to the document.
    #
    #  @param group : [\b str] group name
    #  @param key   : [\b str] key name
    #  @param value : [\b sidex.variant.SDXBase, \b str, \b list, \b tuple, \b dict, \b float,
    #    \b int, \b bool, \b none] value to write
    def write_value(self, group, key, value):
        sdxvalue = sidex.variant.py_to_sdx(value)
        sidex.sidex_Variant_Write(self.__shandle, sidex.variant.sdx_to_py(group), sidex.variant.sdx_to_py(key), sdxvalue.vhandle)

    ## @brief Read a value from the document.
    #
    #  @param group : [\b str] group name
    #  @param key   : [\b str] key name
    #
    #  @return         [\b sidex.variant.SDXBase] value from the document
    def read_value(self, group, key):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_Variant_Read(self.__shandle, sidex.variant.sdx_to_py(group),
                                                                      sidex.variant.sdx_to_py(key)), owner=True)

    ## @brief Save content to file.
    #
    #  @param path : [\b str] filename and path
    def save_to_file(self, path):
        sidex.sidex_Save_Content(self.__shandle, path)

    ## @brief Load content from file.
    #
    #  @param path : [\b str] filename and path
    def load_from_file(self, path):
        sidex.sidex_Load_Content(self.__shandle, path)

    ## @brief Delete all values from the document.
    #
    #  @sa sidex_Clear()
    def clear(self):
        sidex.sidex_Clear(self.__shandle)

    ## @brief Check if a group with \c name exists.
    #
    #  @param name : [\b str] group name
    #
    #  @returns [\b bool] True if the group exists
    #
    #  @sa sidex_HasGroup()
    def has_key(self, name):
        return sidex.sidex_HasGroup(self.__shandle, name)

<commit_msg>inherit MutableMapping for SDXDocument & SDXDocumentGroup<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------------------------------------------
#  libTML:  A BEEP based Messaging Suite
#  Copyright (C) 2015 wobe-systems GmbH
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public License
#  as published by the Free Software Foundation; either version 2.1
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this program; if not, write to the Free
#  Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
#  02111-1307 USA
#
#  You may find a copy of the license under this software is released
#  at COPYING file. This is LGPL software: you are welcome to develop
#  proprietary applications using this library without any royalty or
#  fee but returning back any change, improvement or addition in the
#  form of source code, project image, documentation patches, etc.
#
#  Homepage:
#    http://www.libtml.org
#
#  For professional support contact us:
#
#    wobe-systems GmbH
#    support@libtml.org
#-----------------------------------------------------------

import sidex
import sidex.variant
import collections

from sidex.constants import *

## @defgroup SDXDocGroup SIDEX document
#  @ingroup highLevelClasses
#  @brief Using SIDEX XML documents.
#
#  The SIDEX document manages a group/key based data container. Data types are
#  handled automatically by the document class and the variant data objects to ensure the type is
#  saved together with the data.
#
#  @section SIDEX Document example
#  The example shows the creation of a sidex document and how to add some basic data in the \c General group.
#  The \c Container group contains a list of basic types and other container types like a list
#  and a dictionary. Nesting of container types is the foundation to handle complex data structures.
#
#  @snippet sdx_doc_01.py basic_usage
#
#  The resulting SIDEX XML file contains all data types.
#
#  @include sdx_doc_01_result.xml

## @brief SIDEX parameter group.
#  @ingroup SDXDocGroup
#
#  The class emulates a dictionary to simplify access of the key value pairs
#  in a parameter group.
class SDXDocumentGroup(collections.MutableMapping):

    ## @brief Constructor
    #
    #  @param sdxdoc  :[\b SDXDocument] parent document
    #  @param grpname :[\b str] name of the group
    def __init__(self, sdxdoc, grpname):
        self.__grpname = sidex.variant.sdx_to_py(grpname)
        self.__sdxdoc = sdxdoc

    ## @brief Get an value by key name.
    #
    #  To read a value from a SIDEX document the SDXDocument instance can be used like
    #  a nested dictionary.
    #
    #  @code{.py}
    #  value = sdx_document['General']['ParamKey']
    #  @endcode
    #
    #  @param key : [\b str] name of the key
    #
    #  @returns [\b sidex.variant.SDXBase] the value
    #
    #  @exception KeyError the requested key does not exist
    #
    #  @sa SDXDcoument.__getitem__(), SDXDocument.read_value(), SDXDocument.autocast
    def __getitem__(self, key):
        lkey = sidex.variant.sdx_to_py(key)
        if sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, lkey):
            if self.__sdxdoc.autocast:
                return sidex.variant.sdx_to_py(self.__sdxdoc.read_value(self.__grpname, lkey))
            else:
                return self.__sdxdoc.read_value(self.__grpname, lkey)
        else:
            raise KeyError("invalid key %s" % lkey)

    ## @brief Add or replace a key value pair.
    #
    #  Values can be added to a document by assigning to the double indexed SDXDocument instance.
    #
    #  @code
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  sdoc['MyGroup']['MyKey'] = 3.14
    #  @endcode
    #
    #  @param key   : [\b str] name of the key
    #  @param value : [\b sidex.variant.SDXBase, \b str, \b list, \b tuple, \b dict, \b float,
    #    \b int, \b bool, \b none] value to write
    #
    #  @sa SDXDocument.write_value()
    def __setitem__(self, key, value):
        self.__sdxdoc.write_value(self.__grpname, sidex.variant.sdx_to_py(key), value)

    ## @brief Return an iterable List of keys from the Groupe
    #
    #  @code
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  iter(sdoc['MyGroup'])
    #  @endcode

    #  @returns [\b SDXListIter] iterator over keys
    def __iter__(self):
        return iter(self.keys())

    ## @brief Delete a key value pair from the document.
    #
    #  To Deleta a key/value pair from the document the \c del command
    #  can be used.
    #
    #  @code{.py}
    #  del sdx_document['MyGroup']['MyParamKey']
    #  @endcode
    #
    #  @param key : [\b str] name of the key
    def __delitem__(self, key):
        lkey = sidex.variant.sdx_to_py(key)
        if sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, lkey):
            sidex.sidex_DeleteKey(self.__sdxdoc.shandle, self.__grpname, lkey)

    ## @brief Get a list of key names.
    #
    #  @code{.py}
    #  keynames = sdxdocument['MyGroup'].keys()
    #  @endcode
    #
    #  @returns [\b sidex.variant.SDXList] a list of strings containing the keys
    #
    #  @sa SDXDocumentGroup.keys()
    def keys(self):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_GetKeys(self.__sdxdoc.shandle, self.__grpname))

    ## @brief Get the number of keys in a group.
    #
    #  @returns [\b int] number of keys
    def __len__(self):
        return len(self.keys())

    ## @brief Check if a key with name exists.
    #
    #  @param name : [\b str] key name
    #
    #  @returns [\b bool] True if the key exists
    def has_key(self, name):
        return sidex.sidex_HasKey(self.__sdxdoc.shandle, self.__grpname, sidex.variant.sdx_to_py(name))

## @brief SIDEX document.
#  @ingroup SDXDocGroup
#
#  A SIDEX document can be converted to a SIDEX XML string to be saved as a file or to be send to as a message
#  to another process or host. If a SIDEX file or string has to be parsed, it is simply loaded into a SIDEX
#  document that enables type safe access and modification.
class SDXDocument(collections.MutableMapping):

    ## @brief Constructor
    #
    #  @param document  :[\b int, \b str] pass either the handle of a
    #    sidex document or the name of the sidex document. The name will be used as root node.
    def __init__(self, document=None, content=None):
        if document is None:
            document = "SDXRoot"

        if PY3 and isinstance(document, str):
            self.__thisown = True
            self.__shandle = sidex.sidex_Create(document)
        elif PY2 and isinstance(document, (str, unicode)):
            self.__thisown = True
            self.__shandle = sidex.sidex_Create(document)
        elif PY3 and isinstance(document, int):
            self.__thisown = False
            self.__shandle = document
        elif PY2 and isinstance(document, long):
            self.__thisown = False
            self.__shandle = document
        else:
            raise TypeError("invalid document type %s" % (type(document)))

        if content is not None:
            self.content = content

        self.__autocast = True

        return

        # ------------ properties -----------------
        # the code here is for documentation only
        self.shandle = None
        self.content = ''
        self.document_name = ''
        self.autocast = True
        # ------------ properties -----------------

    ## @brief Destructor of SDXDocument.
    #
    #  If the document object owns the SIDEX handle, it will be freed on destruction.
    def __del__(self):
        if self.__thisown:
            sidex.sidex_Free(self.__shandle)

    ## @property autocast
    #  @brief [\b bool] automatically cast SDX variant objects to python.
    #
    #  If values are read from the document with __getitem__, they are automatically
    #  casted to python objects. If this cast is not necessary assign False to this property.
    #
    #  @par Access:
    #    read/write
    #

    #  @cond NODOC
    @property
    def autocast(self):
        return self.__autocast

    @autocast.setter
    def autocast(self, value):
        self.__autocast = value
    #  @endcond

    ## @property document_name
    #  @brief [\b str] document and root node name in XML data
    #
    #  @par Access:
    #    read/write
    #
    #  @sa sidex_Get_DocumentName(), sidex_Set_DocumentName()

    #  @cond NODOC
    @property
    def document_name(self):
        return sidex.sidex_Get_DocumentName(self.__shandle)

    @document_name.setter
    def document_name(self, value):
        sidex.sidex_Set_DocumentName(self.__shandle, value)
    #  @endcond

    ## @property shandle
    #  @brief [\b None] SIDEX document handle
    #
    #  @par Access:
    #    read

    #  @cond NODOC
    @property
    def shandle(self):
        return self.__shandle
    #  @endcond

    ## @property content
    #  @brief [\b str] Accessing XML content of the document.
    #
    #  @par Access:
    #    read/write
    #
    #  @sa __str__(), sidex_Get_Content(), sidex_Set_Content()

    #  @cond NODOC
    @property
    def content(self):
        return sidex.sidex_Get_Content(self.__shandle)

    @content.setter
    def content(self, value):
        sidex.sidex_Set_Content(self.__shandle, sidex.variant.sdx_to_py(value))
    #  @endcond

    ## @brief Access a parameter group by name.
    #
    #  @param key : [\b str] name of the parameter group
    #
    #  @returns   [\b SDXDocumentGroup] parameter group object
    #
    #  @exception KeyError key does not exist
    def __getitem__(self, key):
        return SDXDocumentGroup(self, key)

    ## @brief Set a group of values.
    #
    #  Setting a group of values directly requires a dictionary with alphanumeric keys matching
    #  the specification for XML tag names.
    #
    #  @code{.py}
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  sdoc['MyGroup'] = {'KeyA':42, 'PI':3.14}
    #  @endcode
    #
    #  An existing group will be replaced.
    #
    #  @param key   : [\b str] name of the group
    #  @param value : [\b dict] dictionary with string keys and values
    #
    #  @exception TypeError group does not match the requirements
    def __setitem__(self, key, value):
        if not isinstance(value, dict):
            raise TypeError("value is not a dictionary")

        for lkey in value.keys():
            if not isinstance(key, str):
                raise TypeError("key in group in not a string %s" % lkey)

        if sidex.sidex_HasGroup(self.__shandle, key):
            self.__delitem__(key)

        for lkey, lvalue in value.items():
            self.write_value(key, lkey, lvalue)

    ## @brief Delete a group from the document.
    #
    #  To delete a complete parameter group from the document the \c del command
    #  can be used.
    #
    #  @code{.py}
    #  del sdx_document['MyGroup']
    #  @endcode
    #
    #  @param key : [\b str] name of the group
    def __delitem__(self, key):
        if sidex.sidex_HasGroup(self.__shandle, key):
            sidex.sidex_DeleteGroup(self.__shandle, key)

    ## @brief Get a list of group names.
    #
    #  The SIDEX document works more a less like a python dict. That's why the function name is \c keys
    #  and not \c groups. In fact it returns a list of group names.
    #
    #  @code{.py}
    #  groupnames = sdxdocument.keys()
    #  @endcode
    #
    #  @returns [\b sidex.variant.SDXList] a list of strings containing the keys
    #
    #  @sa SDXDocumentGroup.keys()
    def keys(self):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_GetGroups(self.__shandle), owner=True)

    ## @brief Get the number of groups in the document.
    #
    #  @returns [\b int] number of groups
    def __len__(self):
        return len(self.keys())

    ## @brief Convert the document data into a SIDEX XML string.
    #
    #  @returns [\b str] SIDEX formatted XML string
    def __str__(self):
        return self.content

    ## @brief Return an iterable List of keys from the SDXDocument
    #
    #  @code
    #  sdoc = sidex.document.SDXDocument('SDXDocTest')
    #  iter(sdoc)
    #  @endcode

    #  @returns [\b SDXListIter] iterator over keys
    def __iter__(self):
        return iter(self.keys())

    ## @brief Write a value to the document.
    #
    #  @param group : [\b str] group name
    #  @param key   : [\b str] key name
    #  @param value : [\b sidex.variant.SDXBase, \b str, \b list, \b tuple, \b dict, \b float,
    #    \b int, \b bool, \b none] value to write
    def write_value(self, group, key, value):
        sdxvalue = sidex.variant.py_to_sdx(value)
        sidex.sidex_Variant_Write(self.__shandle, sidex.variant.sdx_to_py(group), sidex.variant.sdx_to_py(key), sdxvalue.vhandle)

    ## @brief Read a value from the document.
    #
    #  @param group : [\b str] group name
    #  @param key   : [\b str] key name
    #
    #  @return         [\b sidex.variant.SDXBase] value from the document
    def read_value(self, group, key):
        return sidex.variant.wrap_sdx_handle(sidex.sidex_Variant_Read(self.__shandle, sidex.variant.sdx_to_py(group),
                                                                      sidex.variant.sdx_to_py(key)), owner=True)

    ## @brief Save content to file.
    #
    #  @param path : [\b str] filename and path
    def save_to_file(self, path):
        sidex.sidex_Save_Content(self.__shandle, path)

    ## @brief Load content from file.
    #
    #  @param path : [\b str] filename and path
    def load_from_file(self, path):
        sidex.sidex_Load_Content(self.__shandle, path)

    ## @brief Delete all values from the document.
    #
    #  @sa sidex_Clear()
    def clear(self):
        sidex.sidex_Clear(self.__shandle)

    ## @brief Check if a group with \c name exists.
    #
    #  @param name : [\b str] group name
    #
    #  @returns [\b bool] True if the group exists
    #
    #  @sa sidex_HasGroup()
    def has_key(self, name):
        return sidex.sidex_HasGroup(self.__shandle, name)<|endoftext|>


<commit_before>import os
import sys
import subprocess
import xenapi
import lxml
import time
import urllib2

from lxml import etree
from uuid import uuid4

from celery import task

from xenserver.models import XenServer, XenVM

def getSession(hostname, username, password):
    url = 'https://%s:443/' % (hostname)
    # First acquire a valid session by logging in:
    session = xenapi.Session(url)
    session.xenapi.login_with_password(username, password)

    return session

def getHostMetrics(session, hostname):
    t = time.time()-86400

    uri = 'http://%s/rrd_updates?session_id=%s&start=%s&host=true' % (
        hostname, session._session, int(t))

    u = urllib2.urlopen(uri)

    tree = etree.parse(u)

    rows = tree.xpath('/xport/data/row')

    legend = tree.xpath('/xport/meta/legend/entry')

    items = {}
    keys = []

    for i,l in enumerate(legend):
        key = l.text
        items[key] = []
        keys.append(key)

    cpu_all=[]

    for r in rows:
        values = r.xpath('v')
        for i,v in enumerate(values):
            k = keys[i]
            cf, rt, oid, key = k.split(':')
            if rt=='host' and key!='cpu_avg' and key.startswith('cpu'):
                cpu_all.append(float(v.text))

    return int((sum(cpu_all)/len(cpu_all))*100)

@task()
def shutdown_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = shutdown_vm.get_logger()
    logger.info("Stopping %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.shutdown(vm.xsref)
    session.xenapi.session.logout()

@task()
def reboot_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = reboot_vm.get_logger()
    logger.info("Rebooting %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.hard_reboot(vm.xsref)
    session.xenapi.session.logout()

@task()
def start_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = start_vm.get_logger()
    logger.info("Starting %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.start(vm.xsref, False, True)
    session.xenapi.session.logout()

@task()
def destroy_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = destroy_vm.get_logger()
    logger.info("Terminating %s on %s" % (vm.name, xenserver.hostname))

    vmobj = session.xenapi.VM.get_record(vm.xsref)

    try:
        session.xenapi.VM.hard_shutdown(vm.xsref)
    except:
        pass

    # Get attached VBDs and destroy any attached disk VDIs
    vbds = vmobj['VBDs']
    for vbref in vbds:
        vbd = session.xenapi.VBD.get_record(vbref)
        if vbd['type'] == 'Disk':
            vdi = vbd['VDI']
            session.xenapi.VDI.destroy(vdi)

    session.xenapi.VM.destroy(vm.xsref)
    session.xenapi.session.logout()

    vm.delete()

@task
def updateVm(xenserver, vmref, vmobj):
    if (not vmobj['is_a_template']) and (not vmobj['is_control_domain']):
        try:
            session = getSession(xenserver.hostname,
                xenserver.username, xenserver.password)
            netip = session.xenapi.VM_guest_metrics.get_record(
                        vmobj['guest_metrics']
                    )['networks']['0/ip']
            session.xenapi.session.logout()
        except:
            netip = ''

        name = vmobj['name_label']

        try:
            vm = XenVM.objects.get(xenserver=xenserver, name=name)
            vm.name = name
            vm.status = vmobj['power_state']
            vm.sockets = int(vmobj['VCPUs_max'])
            vm.memory = int(vmobj['memory_static_max']) / 1048576
            vm.xenserver = xenserver
            vm.xsref = vmref

            if netip:
                vm.ip = netip

        except XenVM.DoesNotExist:
            vm = XenVM.objects.create(
                xsref=vmref,
                name=name,
                status=vmobj['power_state'],
                sockets=int(vmobj['VCPUs_max']),
                memory=int(vmobj['memory_static_max']) / 1048576,
                xenserver=xenserver,
                ip=netip
            )

        vm.save()

@task
def updateServer(xenserver):
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    # get server info
    host = session.xenapi.host.get_all()[0]
    host_info = session.xenapi.host.get_record(host)
    cores = int(host_info['cpu_info']['cpu_count'])

    xenserver.cores = cores

    metrics = session.xenapi.host_metrics.get_record(
        session.xenapi.host.get_metrics(host)
    )
    memory = metrics['memory_total']
    mem_free = metrics['memory_free']

    xenserver.memory = int(memory) / 1048576
    xenserver.mem_free = int(mem_free) / 1048576
    xenserver.cpu_util = getHostMetrics(session, xenserver.hostname)

    xenserver.save()

    # List all the VM objects
    #allvms = session.xenapi.host.get_resident_VMs(host)
    allvms = session.xenapi.VM.get_all_records()
    vmrefs = allvms.keys()

    session.xenapi.session.logout()

    # Update all the vm info
    for vmref, vmobj in allvms.items():
        updateVm.delay(xenserver, vmref, vmobj)

    # Prevent cleaning an unreferenced VM
    for vm in XenVM.objects.filter(xsref__startswith='TEMPREF'):
        vmrefs.append(vm.xsref)

    # Purge lost VM's
    lost = XenVM.objects.filter(xenserver=xenserver).exclude(xsref__in=vmrefs).delete()

@task()
def updateVms():
    servers = XenServer.objects.all()
    for xenserver in servers:
        updateServer.delay(xenserver)

@task()
def create_vm(xenserver, template, name, domain, ip, subnet, gateway, preseed_url):
    session = getSession(xenserver.hostname,
            xenserver.username, xenserver.password)

    mem_bytes = str(template.memory * (1024*1024))
    cores = str(template.cores)
    disk_bytes = str(template.diskspace * (1024*1024*1024))

    storage = session.xenapi.SR.get_all()
    ubuntu_vdi = ''
    local_sr = ''
    for s in storage:
        store_record = session.xenapi.SR.get_record(s)
        if store_record['type'] == 'iso':
            vdis = store_record['VDIs']
            for vdi in vdis:
                vr = session.xenapi.VDI.get_record(vdi)
                if vr['name_label'] == template.iso:
                    ubuntu_vdi = vdi

        if store_record['name_label'] == 'Local storage':
            # Attatch local SR
            local_sr = s

    if not local_sr:
        raise StorageError("Unable to locate 'Local storage' SR")

    if not ubuntu_vdi:
        raise StorageError('Unable to locate installation source ISO')

    phys = session.xenapi.PIF.get_all()
    network = None
    for phy in phys:
        r = session.xenapi.PIF.get_record(phy)

        if r['gateway']:
            network = phy
            break

    network = session.xenapi.PIF.get_network(network)

    if not network:
        raise NetworkError('Unable to locate VIF network')

    platfrm = {
        'nx': 'true',
        'apic': 'true',
        'viridian': 'true',
        'pae': 'true',
        'acpi': '1'
    }

    boot_params = 'debian-installer/locale=en_ZA.UTF-8 console-setup/layoutcode=us console-setup/ask_detect=false interface=eth0 netcfg/get_hostname=%(name)s netcfg/get_domain=%(domain)s netcfg/disable_autoconfig=true netcfg/get_ipaddress=%(ip)s netcfg/get_netmask=%(subnet)s netcfg/get_gateway=%(gateway)s netcfg/get_nameservers=8.8.8.8 preseed/url=%(url)s' % {
        'ip': ip,
        'subnet': subnet,
        'gateway': gateway,
        'name': name,
        'domain': domain,
        'url': preseed_url
    }

    vmname = "%s.%s" % (name, domain)

    vmprop = {
        'name_label': vmname,
        'name_description': '',
        'user_version': '1',
        'affinity': '',
        'is_a_template': False,
        'auto_power_on': False,
        'memory_static_max': mem_bytes,
        'memory_static_min': mem_bytes,
        'memory_dynamic_max': mem_bytes,
        'memory_dynamic_min': mem_bytes,
        'VCPUs_max': cores,
        'VCPUs_at_startup': cores,
        'VCPUs_params': {},
        'actions_after_shutdown': 'destroy',
        'actions_after_reboot': 'restart',
        'actions_after_crash': 'restart',
        'PV_kernel': '',
        'PV_ramdisk': '',
        'PV_bootloader': 'eliloader',
        'PV_bootloader_args': '',
        'PV_legacy_args': '',
        'HVM_boot_policy': '',
        'HVM_boot_params': {},
        'platform': platfrm,
        'PCI_bus': '',
        'other_config': {
            'auto_poweron': 'true',
            'mac_seed': str(uuid4()),
            'install-distro': 'debianlike',
            'base_template_name': 'Ubuntu Precise Pangolin 12.04 (64-bit)',
            'install-arch': 'amd64',
            'install-methods': 'cdrom,http,ftp',
            'install-repository': 'cdrom',
            'debian-release': 'precise',
            'linux_template': 'true'
        },
        'recommendations': '',
        'PV_args': '-- quiet console=hvc0 %s' % boot_params,
        'suspend_SR': local_sr
    }

    # Create virtual machine
    VM_ref=session.xenapi.VM.create(vmprop)

    vif = { 'device': '0',
            'network': network,
            'VM': VM_ref,
            'MAC': '',
            'MTU': '1500',
            'qos_algorithm_type': '',
            'qos_algorithm_params': {},
            'other_config': {}
        }

    # Create and attach network interface
    session.xenapi.VIF.create(vif)

    vdisk = {
        'name_label': vmname + ' 0',
        'name_description': '',
        'SR': local_sr,
        'virtual_size': disk_bytes,
        'type': 'system',
        'sharable': False,
        'read_only': False,
        'other_config': {}
    }

    # Create the VDI for our disk
    vdi_ref=session.xenapi.VDI.create(vdisk)

    visoconnect = {
        'VDI': ubuntu_vdi,
        'VM': VM_ref,
        'userdevice': '3',
        'allowed_operations': ['attach', 'eject'],
        'storage_lock': False,
        'mode': 'RO',
        'type': 'CD',
        'bootable': True,
        'unpluggable': True,
        'empty': False,
        'other_config': {},
        'qos_algorithm_type': '',
        'qos_algorithm_params': {}
    }
    # Connect the ISO device
    vbdref1=session.xenapi.VBD.create(visoconnect)

    vbdconnect = {
        'VDI': vdi_ref,
        'VM': VM_ref,
        'userdevice': '0',
        'mode': 'RW',
        'type': 'Disk',
        'bootable': False,
        'unpluggable': False,
        'device': '',
        'empty': False,
        'storage_lock': False,
        'other_config': {'owner': ''},
        'qos_algorithm_type': '',
        'qos_algorithm_params': {},
        'allowed_operations': ['attach'],
    }
    # Connect the disk VDI
    vbdref=session.xenapi.VBD.create(vbdconnect)

    # Boot the VM up
    session.xenapi.VM.start(VM_ref, False, False)

    session.xenapi.session.logout()

<commit_msg>Handle NaN values in metrics<commit_after>import os
import sys
import subprocess
import xenapi
import lxml
import time
import urllib2

from lxml import etree
from uuid import uuid4

from celery import task

from xenserver.models import XenServer, XenVM

def getSession(hostname, username, password):
    url = 'https://%s:443/' % (hostname)
    # First acquire a valid session by logging in:
    session = xenapi.Session(url)
    session.xenapi.login_with_password(username, password)

    return session

def getHostMetrics(session, hostname):
    t = time.time()-86400

    uri = 'http://%s/rrd_updates?session_id=%s&start=%s&host=true' % (
        hostname, session._session, int(t))

    u = urllib2.urlopen(uri)

    tree = etree.parse(u)

    rows = tree.xpath('/xport/data/row')

    legend = tree.xpath('/xport/meta/legend/entry')

    items = {}
    keys = []

    for i,l in enumerate(legend):
        key = l.text
        items[key] = []
        keys.append(key)

    cpu_all=[]

    for r in rows:
        values = r.xpath('v')
        for i,v in enumerate(values):
            k = keys[i]
            cf, rt, oid, key = k.split(':')
            if rt=='host' and key!='cpu_avg' and key.startswith('cpu'):
                if v.text == 'NaN':
                    continue
                cpu_all.append(float(v.text))

    return int((sum(cpu_all)/len(cpu_all))*100)

@task()
def shutdown_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = shutdown_vm.get_logger()
    logger.info("Stopping %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.shutdown(vm.xsref)
    session.xenapi.session.logout()

@task()
def reboot_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = reboot_vm.get_logger()
    logger.info("Rebooting %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.hard_reboot(vm.xsref)
    session.xenapi.session.logout()

@task()
def start_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = start_vm.get_logger()
    logger.info("Starting %s on %s" % (vm.name, xenserver.hostname))

    session.xenapi.VM.start(vm.xsref, False, True)
    session.xenapi.session.logout()

@task()
def destroy_vm(vm):
    xenserver = vm.xenserver
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    logger = destroy_vm.get_logger()
    logger.info("Terminating %s on %s" % (vm.name, xenserver.hostname))

    vmobj = session.xenapi.VM.get_record(vm.xsref)

    try:
        session.xenapi.VM.hard_shutdown(vm.xsref)
    except:
        pass

    # Get attached VBDs and destroy any attached disk VDIs
    vbds = vmobj['VBDs']
    for vbref in vbds:
        vbd = session.xenapi.VBD.get_record(vbref)
        if vbd['type'] == 'Disk':
            vdi = vbd['VDI']
            session.xenapi.VDI.destroy(vdi)

    session.xenapi.VM.destroy(vm.xsref)
    session.xenapi.session.logout()

    vm.delete()

@task
def updateVm(xenserver, vmref, vmobj):
    if (not vmobj['is_a_template']) and (not vmobj['is_control_domain']):
        try:
            session = getSession(xenserver.hostname,
                xenserver.username, xenserver.password)
            netip = session.xenapi.VM_guest_metrics.get_record(
                        vmobj['guest_metrics']
                    )['networks']['0/ip']
            session.xenapi.session.logout()
        except:
            netip = ''

        name = vmobj['name_label']

        try:
            vm = XenVM.objects.get(xenserver=xenserver, name=name)
            vm.name = name
            vm.status = vmobj['power_state']
            vm.sockets = int(vmobj['VCPUs_max'])
            vm.memory = int(vmobj['memory_static_max']) / 1048576
            vm.xenserver = xenserver
            vm.xsref = vmref

            if netip:
                vm.ip = netip

        except XenVM.DoesNotExist:
            vm = XenVM.objects.create(
                xsref=vmref,
                name=name,
                status=vmobj['power_state'],
                sockets=int(vmobj['VCPUs_max']),
                memory=int(vmobj['memory_static_max']) / 1048576,
                xenserver=xenserver,
                ip=netip
            )

        vm.save()

@task
def updateServer(xenserver):
    session = getSession(xenserver.hostname,
        xenserver.username, xenserver.password)

    # get server info
    host = session.xenapi.host.get_all()[0]
    host_info = session.xenapi.host.get_record(host)
    cores = int(host_info['cpu_info']['cpu_count'])

    xenserver.cores = cores

    metrics = session.xenapi.host_metrics.get_record(
        session.xenapi.host.get_metrics(host)
    )
    memory = metrics['memory_total']
    mem_free = metrics['memory_free']

    xenserver.memory = int(memory) / 1048576
    xenserver.mem_free = int(mem_free) / 1048576

    try:
        xenserver.cpu_util = getHostMetrics(session, xenserver.hostname)
    except:
        xenserver.cpu_util = 0

    xenserver.save()

    # List all the VM objects
    #allvms = session.xenapi.host.get_resident_VMs(host)
    allvms = session.xenapi.VM.get_all_records()
    vmrefs = allvms.keys()

    session.xenapi.session.logout()

    # Update all the vm info
    for vmref, vmobj in allvms.items():
        updateVm.delay(xenserver, vmref, vmobj)

    # Prevent cleaning an unreferenced VM
    for vm in XenVM.objects.filter(xsref__startswith='TEMPREF'):
        vmrefs.append(vm.xsref)

    # Purge lost VM's
    lost = XenVM.objects.filter(xenserver=xenserver).exclude(xsref__in=vmrefs).delete()

@task()
def updateVms():
    servers = XenServer.objects.all()
    for xenserver in servers:
        updateServer.delay(xenserver)

@task()
def create_vm(xenserver, template, name, domain, ip, subnet, gateway, preseed_url):
    session = getSession(xenserver.hostname,
            xenserver.username, xenserver.password)

    mem_bytes = str(template.memory * (1024*1024))
    cores = str(template.cores)
    disk_bytes = str(template.diskspace * (1024*1024*1024))

    storage = session.xenapi.SR.get_all()
    ubuntu_vdi = ''
    local_sr = ''
    for s in storage:
        store_record = session.xenapi.SR.get_record(s)
        if store_record['type'] == 'iso':
            vdis = store_record['VDIs']
            for vdi in vdis:
                vr = session.xenapi.VDI.get_record(vdi)
                if vr['name_label'] == template.iso:
                    ubuntu_vdi = vdi

        if store_record['name_label'] == 'Local storage':
            # Attatch local SR
            local_sr = s

    if not local_sr:
        raise StorageError("Unable to locate 'Local storage' SR")

    if not ubuntu_vdi:
        raise StorageError('Unable to locate installation source ISO')

    phys = session.xenapi.PIF.get_all()
    network = None
    for phy in phys:
        r = session.xenapi.PIF.get_record(phy)

        if r['gateway']:
            network = phy
            break

    network = session.xenapi.PIF.get_network(network)

    if not network:
        raise NetworkError('Unable to locate VIF network')

    platfrm = {
        'nx': 'true',
        'apic': 'true',
        'viridian': 'true',
        'pae': 'true',
        'acpi': '1'
    }

    boot_params = 'debian-installer/locale=en_ZA.UTF-8 console-setup/layoutcode=us console-setup/ask_detect=false interface=eth0 netcfg/get_hostname=%(name)s netcfg/get_domain=%(domain)s netcfg/disable_autoconfig=true netcfg/get_ipaddress=%(ip)s netcfg/get_netmask=%(subnet)s netcfg/get_gateway=%(gateway)s netcfg/get_nameservers=8.8.8.8 preseed/url=%(url)s' % {
        'ip': ip,
        'subnet': subnet,
        'gateway': gateway,
        'name': name,
        'domain': domain,
        'url': preseed_url
    }

    vmname = "%s.%s" % (name, domain)

    vmprop = {
        'name_label': vmname,
        'name_description': '',
        'user_version': '1',
        'affinity': '',
        'is_a_template': False,
        'auto_power_on': False,
        'memory_static_max': mem_bytes,
        'memory_static_min': mem_bytes,
        'memory_dynamic_max': mem_bytes,
        'memory_dynamic_min': mem_bytes,
        'VCPUs_max': cores,
        'VCPUs_at_startup': cores,
        'VCPUs_params': {},
        'actions_after_shutdown': 'destroy',
        'actions_after_reboot': 'restart',
        'actions_after_crash': 'restart',
        'PV_kernel': '',
        'PV_ramdisk': '',
        'PV_bootloader': 'eliloader',
        'PV_bootloader_args': '',
        'PV_legacy_args': '',
        'HVM_boot_policy': '',
        'HVM_boot_params': {},
        'platform': platfrm,
        'PCI_bus': '',
        'other_config': {
            'auto_poweron': 'true',
            'mac_seed': str(uuid4()),
            'install-distro': 'debianlike',
            'base_template_name': 'Ubuntu Precise Pangolin 12.04 (64-bit)',
            'install-arch': 'amd64',
            'install-methods': 'cdrom,http,ftp',
            'install-repository': 'cdrom',
            'debian-release': 'precise',
            'linux_template': 'true'
        },
        'recommendations': '',
        'PV_args': '-- quiet console=hvc0 %s' % boot_params,
        'suspend_SR': local_sr
    }

    # Create virtual machine
    VM_ref=session.xenapi.VM.create(vmprop)

    vif = { 'device': '0',
            'network': network,
            'VM': VM_ref,
            'MAC': '',
            'MTU': '1500',
            'qos_algorithm_type': '',
            'qos_algorithm_params': {},
            'other_config': {}
        }

    # Create and attach network interface
    session.xenapi.VIF.create(vif)

    vdisk = {
        'name_label': vmname + ' 0',
        'name_description': '',
        'SR': local_sr,
        'virtual_size': disk_bytes,
        'type': 'system',
        'sharable': False,
        'read_only': False,
        'other_config': {}
    }

    # Create the VDI for our disk
    vdi_ref=session.xenapi.VDI.create(vdisk)

    visoconnect = {
        'VDI': ubuntu_vdi,
        'VM': VM_ref,
        'userdevice': '3',
        'allowed_operations': ['attach', 'eject'],
        'storage_lock': False,
        'mode': 'RO',
        'type': 'CD',
        'bootable': True,
        'unpluggable': True,
        'empty': False,
        'other_config': {},
        'qos_algorithm_type': '',
        'qos_algorithm_params': {}
    }
    # Connect the ISO device
    vbdref1=session.xenapi.VBD.create(visoconnect)

    vbdconnect = {
        'VDI': vdi_ref,
        'VM': VM_ref,
        'userdevice': '0',
        'mode': 'RW',
        'type': 'Disk',
        'bootable': False,
        'unpluggable': False,
        'device': '',
        'empty': False,
        'storage_lock': False,
        'other_config': {'owner': ''},
        'qos_algorithm_type': '',
        'qos_algorithm_params': {},
        'allowed_operations': ['attach'],
    }
    # Connect the disk VDI
    vbdref=session.xenapi.VBD.create(vbdconnect)

    # Boot the VM up
    session.xenapi.VM.start(VM_ref, False, False)

    session.xenapi.session.logout()

<|endoftext|>


<commit_before>import copy
import warnings

from .compat import json
import sigopt.vendored.six as six

class ListOf(object):
  def __init__(self, type):
    self.type = type

  def __call__(self, value):
    return [self.type(v) for v in value]

Any = lambda x: x

class Field(object):
  def __init__(self, type):
    self.type = type

  def __call__(self, value):
    if value is None:
      return None
    return self.type(value)

class DeprecatedField(Field):
  def __init__(self, type, recommendation=None):
    super(DeprecatedField, self).__init__(type)
    self.recommendation = (' ' + recommendation) if recommendation else ''

  def __call__(self, value):
    warnings.warn(
      'This field has been deprecated and may be removed in a future version.{0}'.format(self.recommendation),
      DeprecationWarning,
    )
    return super(DeprecatedField, self).__call__(value)

class BaseApiObject(object):
  def __getattribute__(self, name):
    value = object.__getattribute__(self, name)
    if isinstance(value, Field):
      return value(self._body.get(name))
    return value

  def __setattr__(self, name, value):
    field = self._get_field(name)
    if field:
      value = ApiObject.as_json(value)
      self._body[name] = value
    else:
      object.__setattr__(self, name, value)

  def __delattr__(self, name):
    field = self._get_field(name)
    if field:
      del self._body[name]
    else:
      object.__delattr__(self, name)

  def _get_field(self, name):
    try:
      subvalue = object.__getattribute__(self, name)
    except AttributeError:
      return None
    else:
      return subvalue if isinstance(subvalue, Field) else None

  def __repr__(self):
    return six.u('{0}({1})').format(
      self.__class__.__name__,
      json.dumps(
        ApiObject.as_json(self._body),
        indent=2,
        sort_keys=True,
        separators=(',',': '),
      ),
    )

  def to_json(self):
    return copy.deepcopy(self._body)

class ApiObject(BaseApiObject):
  def __init__(self, body, bound_endpoint=None, retrieve_params=None):
    super(ApiObject, self).__init__()
    object.__setattr__(self, '_body', body)
    object.__setattr__(self, '_bound_endpoint', bound_endpoint)
    object.__setattr__(self, '_retrieve_params', retrieve_params)

  def __eq__(self, other):
    return (
      isinstance(other, self.__class__) and
      self._body == other._body
    )

  @staticmethod
  def as_json(obj):
    if isinstance(obj, BaseApiObject):
      return obj.to_json()
    elif isinstance(obj, dict):
      c = {}
      for key in obj:
        c[key] = ApiObject.as_json(obj[key])
      return c
    elif isinstance(obj, list):
      return [ApiObject.as_json(c) for c in obj]
    else:
      return obj

class _DictWrapper(BaseApiObject, dict):
  def __init__(self, body, bound_endpoint=None, retrieve_params=None):
    super(_DictWrapper, self).__init__()
    dict.__init__(self, body)
    self._bound_endpoint = bound_endpoint
    self._retrieve_params = retrieve_params

  @property
  def _body(self):
    return self

  def to_json(self):
    return dict(copy.deepcopy(self))

  def copy(self):
    return self.__class__(dict.copy(self))

  def __eq__(self, other):
    return (
      isinstance(other, self.__class__) and
      dict.__eq__(self, other)
    )

class Assignments(_DictWrapper):
  pass

class Bounds(ApiObject):
  max = Field(float)
  min = Field(float)

class CategoricalValue(ApiObject):
  enum_index = Field(int)
  name = Field(six.text_type)

class Client(ApiObject):
  created = Field(int)
  id = Field(six.text_type)
  name = Field(six.text_type)

class Metadata(_DictWrapper):
  pass

class MetricEvaluation(ApiObject):
  name = Field(six.text_type)
  value = Field(float)
  value_stddev = Field(float)

class Metric(ApiObject):
  name = Field(six.text_type)

class Observation(ApiObject):
  assignments = Field(Assignments)
  created = Field(int)
  experiment = Field(six.text_type)
  failed = Field(bool)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  suggestion = Field(six.text_type)
  value = Field(float)
  value_stddev = Field(float)
  values = Field(ListOf(MetricEvaluation))

class Paging(ApiObject):
  after = Field(six.text_type)
  before = Field(six.text_type)

class Pagination(ApiObject):
  count = Field(int)
  paging = Field(Paging)

  def __init__(self, data_cls, body, bound_endpoint=None, retrieve_params=None):
    super(Pagination, self).__init__(body, bound_endpoint, retrieve_params)
    self.data_cls = data_cls

  @property
  def data(self):
    return Field(ListOf(self.data_cls))(self._body.get('data'))

  def iterate_pages(self):
    data = self.data
    paging = self.paging or Paging({})
    use_before = bool(paging.before)
    while data:
      for d in data:
        yield d
      next_paging = dict(before=paging.before) if use_before else dict(after=paging.after)
      if next_paging.get('before') is not None or next_paging.get('after') is not None:
        params = self._retrieve_params.copy()
        params.pop('before', None)
        params.pop('after', None)
        params.update(next_paging)
        response = self._bound_endpoint(**params)
        data = response.data
        paging = response.paging
      else:
        data = []
        paging = None

class Parameter(ApiObject):
  bounds = Field(Bounds)
  categorical_values = Field(ListOf(CategoricalValue))
  default_value = Field(Any)
  name = Field(six.text_type)
  precision = Field(int)
  tunable = DeprecatedField(bool)
  type = Field(six.text_type)

class PlanPeriod(ApiObject):
  end = Field(int)
  experiments = Field(ListOf(six.text_type))
  start = Field(int)

class PlanRules(ApiObject):
  max_categorical_breadth = Field(int)
  max_dimension = Field(int)
  max_experiments = Field(int)
  max_observations = Field(int)
  max_parallelism = Field(int)
  max_users = Field(int)

class Plan(ApiObject):
  current_period = Field(PlanPeriod)
  id = Field(six.text_type)
  name = Field(six.text_type)
  rules = Field(PlanRules)

class Progress(ApiObject):
  best_observation = DeprecatedField(Observation, recommendation='Prefer the `best_assignments` endpoint')
  first_observation = Field(Observation)
  last_observation = Field(Observation)
  observation_count = Field(int)

class Suggestion(ApiObject):
  assignments = Field(Assignments)
  created = Field(int)
  experiment = Field(six.text_type)
  fold = Field(six.text_type)
  fold_index = Field(int)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  state = Field(six.text_type)

class Experiment(ApiObject):
  can_be_deleted = DeprecatedField(bool)
  client = Field(six.text_type)
  created = Field(int)
  development = Field(bool)
  folds = Field(int)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  metric = Field(Metric)
  metrics = Field(ListOf(Metric))
  name = Field(six.text_type)
  observation_budget = Field(int)
  parameters = Field(ListOf(Parameter))
  progress = Field(Progress)
  state = Field(six.text_type)
  type = Field(six.text_type)

class Token(ApiObject):
  all_experiments = Field(bool)
  development = Field(bool)
  permissions = Field(six.text_type)
  token = Field(six.text_type)
  client = Field(six.text_type)
  experiment = Field(six.text_type)
  user = Field(six.text_type)

class BestAssignments(ApiObject):
  assignments = Field(Assignments)
  value = Field(float)
  value_stddev = Field(float)
  values = Field(ListOf(MetricEvaluation))
<commit_msg>Add missing field<commit_after>import copy
import warnings

from .compat import json
import sigopt.vendored.six as six

class ListOf(object):
  def __init__(self, type):
    self.type = type

  def __call__(self, value):
    return [self.type(v) for v in value]

Any = lambda x: x

class Field(object):
  def __init__(self, type):
    self.type = type

  def __call__(self, value):
    if value is None:
      return None
    return self.type(value)

class DeprecatedField(Field):
  def __init__(self, type, recommendation=None):
    super(DeprecatedField, self).__init__(type)
    self.recommendation = (' ' + recommendation) if recommendation else ''

  def __call__(self, value):
    warnings.warn(
      'This field has been deprecated and may be removed in a future version.{0}'.format(self.recommendation),
      DeprecationWarning,
    )
    return super(DeprecatedField, self).__call__(value)

class BaseApiObject(object):
  def __getattribute__(self, name):
    value = object.__getattribute__(self, name)
    if isinstance(value, Field):
      return value(self._body.get(name))
    return value

  def __setattr__(self, name, value):
    field = self._get_field(name)
    if field:
      value = ApiObject.as_json(value)
      self._body[name] = value
    else:
      object.__setattr__(self, name, value)

  def __delattr__(self, name):
    field = self._get_field(name)
    if field:
      del self._body[name]
    else:
      object.__delattr__(self, name)

  def _get_field(self, name):
    try:
      subvalue = object.__getattribute__(self, name)
    except AttributeError:
      return None
    else:
      return subvalue if isinstance(subvalue, Field) else None

  def __repr__(self):
    return six.u('{0}({1})').format(
      self.__class__.__name__,
      json.dumps(
        ApiObject.as_json(self._body),
        indent=2,
        sort_keys=True,
        separators=(',',': '),
      ),
    )

  def to_json(self):
    return copy.deepcopy(self._body)

class ApiObject(BaseApiObject):
  def __init__(self, body, bound_endpoint=None, retrieve_params=None):
    super(ApiObject, self).__init__()
    object.__setattr__(self, '_body', body)
    object.__setattr__(self, '_bound_endpoint', bound_endpoint)
    object.__setattr__(self, '_retrieve_params', retrieve_params)

  def __eq__(self, other):
    return (
      isinstance(other, self.__class__) and
      self._body == other._body
    )

  @staticmethod
  def as_json(obj):
    if isinstance(obj, BaseApiObject):
      return obj.to_json()
    elif isinstance(obj, dict):
      c = {}
      for key in obj:
        c[key] = ApiObject.as_json(obj[key])
      return c
    elif isinstance(obj, list):
      return [ApiObject.as_json(c) for c in obj]
    else:
      return obj

class _DictWrapper(BaseApiObject, dict):
  def __init__(self, body, bound_endpoint=None, retrieve_params=None):
    super(_DictWrapper, self).__init__()
    dict.__init__(self, body)
    self._bound_endpoint = bound_endpoint
    self._retrieve_params = retrieve_params

  @property
  def _body(self):
    return self

  def to_json(self):
    return dict(copy.deepcopy(self))

  def copy(self):
    return self.__class__(dict.copy(self))

  def __eq__(self, other):
    return (
      isinstance(other, self.__class__) and
      dict.__eq__(self, other)
    )

class Assignments(_DictWrapper):
  pass

class Bounds(ApiObject):
  max = Field(float)
  min = Field(float)

class CategoricalValue(ApiObject):
  enum_index = Field(int)
  name = Field(six.text_type)

class Client(ApiObject):
  created = Field(int)
  id = Field(six.text_type)
  name = Field(six.text_type)

class Metadata(_DictWrapper):
  pass

class MetricEvaluation(ApiObject):
  name = Field(six.text_type)
  value = Field(float)
  value_stddev = Field(float)

class Metric(ApiObject):
  name = Field(six.text_type)

class Observation(ApiObject):
  assignments = Field(Assignments)
  created = Field(int)
  experiment = Field(six.text_type)
  failed = Field(bool)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  suggestion = Field(six.text_type)
  value = Field(float)
  value_stddev = Field(float)
  values = Field(ListOf(MetricEvaluation))

class Paging(ApiObject):
  after = Field(six.text_type)
  before = Field(six.text_type)

class Pagination(ApiObject):
  count = Field(int)
  paging = Field(Paging)

  def __init__(self, data_cls, body, bound_endpoint=None, retrieve_params=None):
    super(Pagination, self).__init__(body, bound_endpoint, retrieve_params)
    self.data_cls = data_cls

  @property
  def data(self):
    return Field(ListOf(self.data_cls))(self._body.get('data'))

  def iterate_pages(self):
    data = self.data
    paging = self.paging or Paging({})
    use_before = bool(paging.before)
    while data:
      for d in data:
        yield d
      next_paging = dict(before=paging.before) if use_before else dict(after=paging.after)
      if next_paging.get('before') is not None or next_paging.get('after') is not None:
        params = self._retrieve_params.copy()
        params.pop('before', None)
        params.pop('after', None)
        params.update(next_paging)
        response = self._bound_endpoint(**params)
        data = response.data
        paging = response.paging
      else:
        data = []
        paging = None

class Parameter(ApiObject):
  bounds = Field(Bounds)
  categorical_values = Field(ListOf(CategoricalValue))
  default_value = Field(Any)
  name = Field(six.text_type)
  precision = Field(int)
  tunable = DeprecatedField(bool)
  type = Field(six.text_type)

class PlanPeriod(ApiObject):
  end = Field(int)
  experiments = Field(ListOf(six.text_type))
  start = Field(int)

class PlanRules(ApiObject):
  max_categorical_breadth = Field(int)
  max_dimension = Field(int)
  max_experiments = Field(int)
  max_metrics = Field(int)
  max_observations = Field(int)
  max_parallelism = Field(int)
  max_users = Field(int)

class Plan(ApiObject):
  current_period = Field(PlanPeriod)
  id = Field(six.text_type)
  name = Field(six.text_type)
  rules = Field(PlanRules)

class Progress(ApiObject):
  best_observation = DeprecatedField(Observation, recommendation='Prefer the `best_assignments` endpoint')
  first_observation = Field(Observation)
  last_observation = Field(Observation)
  observation_count = Field(int)

class Suggestion(ApiObject):
  assignments = Field(Assignments)
  created = Field(int)
  experiment = Field(six.text_type)
  fold = Field(six.text_type)
  fold_index = Field(int)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  state = Field(six.text_type)

class Experiment(ApiObject):
  can_be_deleted = DeprecatedField(bool)
  client = Field(six.text_type)
  created = Field(int)
  development = Field(bool)
  folds = Field(int)
  id = Field(six.text_type)
  metadata = Field(Metadata)
  metric = Field(Metric)
  metrics = Field(ListOf(Metric))
  name = Field(six.text_type)
  observation_budget = Field(int)
  parameters = Field(ListOf(Parameter))
  progress = Field(Progress)
  state = Field(six.text_type)
  type = Field(six.text_type)

class Token(ApiObject):
  all_experiments = Field(bool)
  development = Field(bool)
  permissions = Field(six.text_type)
  token = Field(six.text_type)
  client = Field(six.text_type)
  experiment = Field(six.text_type)
  user = Field(six.text_type)

class BestAssignments(ApiObject):
  assignments = Field(Assignments)
  value = Field(float)
  value_stddev = Field(float)
  values = Field(ListOf(MetricEvaluation))
<|endoftext|>


<commit_before>#! /usr/bin/env python
import os
import cmd
import readline
import datetime
from csv import *
from tqdm import tqdm
from tabulate import tabulate
from pyfiglet import Figlet
from isbntools.app import meta
from tinydb import TinyDB, Query
from fuzzywuzzy import fuzz, process
from tinydb.operations import delete
from collections import defaultdict
##########################################################
def _concat(data):
    final = defaultdict(list)
    for d in data:
        for key, value in d.iteritems():
            final[key].append(value)
    return final
def _cleanify(bookMeta):
    if type(bookMeta['Authors']) == list:
        if len(bookMeta['Authors']) > 1:
            bookMeta['Authors'] = bookMeta['Authors'][0]
        else:
            bookMeta['Authors'] = bookMeta['Authors'][0]
    bookMeta['ISBN'] = bookMeta.pop('ISBN-13')
    title = bookMeta['Title']
    title = title.split(":")[0]
    bookMeta['Title'] = title
    auth = bookMeta['Authors']
    auth = auth.split(",")[0]
    auth = auth.split(";")[0]
    auth = auth.split("illustrated", 1)[0]
    auth = auth.split("and", 1)[0]
    auth = auth.split("with", 1)[0]
    bookMeta['Authors'] = auth
    pub = bookMeta['Publisher']
    pub = pub.split(" ")[0]
    bookMeta['Publisher'] = pub
    return bookMeta
def _doesexist(db, isbn):
    ext = Query()
    if db.search(ext.ISBN.matches(isbn)):
        return True
    else:
        return False
def _today():
    dat = datetime.date.today()
    return dat.strftime('%d/%m/%y')
_intro = Figlet(font = 'slant').renderText('xLibris')
############################################################
class xlibris(object):
    def __init__(self, directory = 'xlibris/'):
        self.directory = directory
        if not os.path.exists(self.directory):
            os.mkdir(self.directory)
    def new(self,dbname):
        '''Creates a new database assigns it to a current database instance to be used by all the other functions
            Usage:-
            new('database name') or new database_name
        '''
        self.db = TinyDB(self.directory + dbname + '.json')
        print 'New database {} created at {}'.format(dbname, self.directory + dbname + '.json' )
    def connect(self,name):
        '''Connect to an existing database for updating/Query
            Usage:-
            connect('name') or connect name
            where 'name' is the name of the existing database'''

        self.db = TinyDB(self.directory + name + '.json')
        print 'Connected to {}'.format(name)
    def display(self):
        try:
            print tabulate(_concat(self.db.all()), headers='keys', tablefmt="simple")
            print "\n"
            print self.count()
        except AttributeError:
            print 'No database Connected'
    def add(self, ISBN):
        '''Add books to the current working database
            Usage:-
            add(ISBN) or add ISBN'''
        if _doesexist(self.db, ISBN) == False:
            try:
                bookData = meta(ISBN)
                bookData = _cleanify(bookData)
                bookData['Date Added'] = _today()
                self.db.insert(bookData)
                print 'ISBN {} inserted'.format(ISBN)
            except:
                print 'ISBN {} not found. Please add details manually- '.format(ISBN)
                self.madd()
        else:
            print 'Book Already Exists'
    def madd(self):
        bookData = {}
        bookData['Authors'] = raw_input('Authors Name: ')
        bookData['ISBN'] = raw_input('ISBN: ')
        bookData['Language'] = raw_input('Language: ')
        bookData['Publisher'] = raw_input('Publisher: ')
        bookData['Title'] = raw_input('Title: ')
        bookData['Year'] = raw_input('Year: ')
        bookData['Date Added'] = _today()
        self.db.insert(bookData)
    def search(self, keyword):
        NewSearch = Query()
        title = self.db.search(NewSearch.Title == keyword)
        auth = self.db.search(NewSearch.Authors == keyword)
        pub = self.db.search(NewSearch.Publisher == keyword)
        isbn = self.db.search(NewSearch.ISBN == keyword)
        ttable = [title, auth, pub, isbn]

        for i in ttable:
            if i:
                print 'Matches Found for {}'.format(keyword)
                print tabulate(_concat(i), headers='keys', tablefmt="simple")
    def _blkadd(self, ISBNlist):
        with tqdm(ISBNlist) as pbar:
            for i in pbar:
                pbar.set_description("Adding %s "%i)
                self.add(i)
                pbar.update(1/len(ISBNlist)*100)
    def add_from_file(self, filename):
        with open(filename, 'rb') as f:
            raw = reader(f)
            final = list(raw)
        for i in range(len(final)):
            final[i] = str(final[i][0])
        self._blkadd(final)
        print 'Done'
    def change_title(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Title ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change title of :- {}'.format(title['Title'])
        self.db.update(change('Title'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def change_author(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Author ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change author of :- {}'.format(title['Title'])
        self.db.update(change('Authors'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def change_publisher(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Publisher ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change Publisher of :- {}'.format(title['Title'])
        self.db.update(change('Publisher'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def write_to_file(self, filename):
        try:
            data = tabulate(_concat(self.db.all()), headers='keys', tablefmt="simple")
        except AttributeError:
            print 'No database Connected'
        f = open('xlibris/' + filename + '.txt', 'w')
        f.write(data.encode('utf8'))
        f.close()
        print 'Written to {}'.format('xlibris/' + filename + '.txt')
    def purge_current(self):
        self.db.purge()
    def remove(self, isbn):
        tmp = Query()
        resp = raw_input('Delete \n {} \n ? (y/n)'.format(tabulate(self.db.search(tmp.ISBN == isbn), headers='keys')))
        resp = resp.lower()
        if resp == 'y':
            for i in ['Publisher', 'Title', 'Authors', 'Year', 'Date Added', 'Language', 'ISBN']:
                self.db.update(delete(i), tmp.ISBN == isbn)
            print 'Deleted'
        elif resp == 'n':
            print 'Spared'
    def lookup(self, keyword):
        data = _concat(self.db.all())
        title = data.pop('Title')
        auth = data.pop('Authors')
        choices = title + auth
        searchKey = process.extractOne(keyword, choices)[0]
        self.search(searchKey)
    def count(self):
        listisbn = _concat(self.db.all())
        listisbn = listisbn.pop('ISBN')
        return "Total {} books.".format(len(listisbn))
##############CONSOLE##############################
class xlibris_repl(cmd.Cmd):
    def __init__(self):
        cmd.Cmd.__init__(self)
        self.prompt = 'xLibris>> '
        self.intro = _intro + '''A small library management system for your books written in python.
        Connect to a database and then just type the ISBN to add a book.
         Type help to find help
        Author - Shubham Bhushan
        '''
        self.current_db = xlibris()
        self.lst = ''
    def do_hist(self, args):
        '''Prints the list of commands entered recently'''
        print self._hist
    def do_exit(self, args):
        '''Exists from console after you type exit'''
        return -1
    def do_EOF(self, args):
        '''Exit on EOF character'''
        return True
    def do_list_db(self, args):
        '''Lists the databases in the native folder.
        Databases are stored as .json files
        simply type list_db to view available databases'''
        # listdb += [i for i in os.listdir('xlibris/') if i.endswith('.json')]
        listdb = {}
        dbs = []
        for i in os.listdir('xlibris/'):
            if i.endswith('.json'):
                dbs.append(i)
        listdb['Databases'] = dbs
        print tabulate(listdb, headers='keys', tablefmt='fancy_grid')
    def do_help(self, args):
        '''Get help on commands.
        type help for a list of available commands
        or help <command> for instructions on a specific command'''
        cmd.Cmd.do_help(self, args)
    def preloop(self):
        cmd.Cmd.preloop(self)
        self._hist    = []
        self._locals  = {}
        self._globals = {}
    def postloop(self):
        cmd.Cmd.postloop(self)
        print 'Exiting xLibris... '
    def precmd(self, args):
        self._hist += [args.split()]
        return args
    def do_new(self, args):
        '''Creates a new database.
        new <name> where <name> is the name of db you wish to create.'''
        self.current_db.new(args)
    def do_connect(self, args):
        '''Connect to an existing database
        connect <name of database without .json>'''
        self.current_db.connect(args)
    def do_add(self, args):
        '''Adds books to the current database.
        add <isbn> to add a book.'''
        self.current_db.add(args)
    def default(self, args):
        '''Adds the book to current database when just ISBN is typed'''
        try:
            self.current_db.add(str(int(args)))
        except ValueError:
            print "Only ISBN, to see a list of functions type help"
    def do_display(self, args):
        '''Prints the current database to screen'''
        self.current_db.display()
    def do_search(self, args):
        '''Searches the database for given keyword.
        Usage
        search <keyword>
        legal keywords include ISBN, author, Title. '''
        self.current_db.search(args)
    def do_lookup(self, args):
        '''Fuzzy Searching can search from partial titles or Author Names
        lookup <keyword>'''
        self.current_db.lookup(args)
    def do_addFromFile(self, args):
        '''Add books from a csv file containing ISBNs.
        type add_from_file <path/filename.csv>'''
        self.current_db.add_from_file(args)
    def do_writeToFile(self, args):
        '''write the current database in a formatted table to a file on the disk
        writeToFile <filename>'''
        self.current_db.write_to_file(args)
    def do_purgeCurrent(self, args):
        '''Purges current database. Beware.
        type purgeCurrent'''
        self.current_db.purge_current()
    def do_changeTitle(self, args):
        '''Helps you change title of the book you have added.
        changeTitle ISBN to change the title of the book'''
        try:
            self.current_db.change_title(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def do_changeAuthor(self, args):
        '''Helps you change the Author of the book you have added.
        changeAuthor ISBN to change the author of the book'''
        try:
            self.current_db.change_author(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def do_changePublisher(self, args):
        '''Helps you change the Publisher of the book you have added.
        changeAuthor ISBN to change the author of the book'''
        try:
            self.current_db.change_publisher(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def emptyline(self):
        print '''Create a new database using new or connect to existing database using connect
        or type help to see a list of available commands. You can connect to one of the following
        '''
        self.do_list_db('foo')
    def do_shell(self, args):
        '''Use shell commands by affixing them with ! sign'''
        os.system(args)
    def do_remove(self, args):
        '''Remove a book from the database.
        remove <isbn>
        * ALthough I have no idea why anyone would use this heinous function'''
        self.current_db.remove(args)
#TODO - Add remove function
if __name__ == '__main__':
    xlibris_repl().cmdloop()
<commit_msg>played around with fuzzy string matching<commit_after>#! /usr/bin/env python
import os
import cmd
import readline
import datetime
from csv import *
from tqdm import tqdm
from tabulate import tabulate
from pyfiglet import Figlet
from isbntools.app import meta
from tinydb import TinyDB, Query
from fuzzywuzzy import fuzz, process
from tinydb.operations import delete
from collections import defaultdict
##########################################################
def _concat(data):
    final = defaultdict(list)
    for d in data:
        for key, value in d.iteritems():
            final[key].append(value)
    return final
def _cleanify(bookMeta):
    if type(bookMeta['Authors']) == list:
        if len(bookMeta['Authors']) > 1:
            bookMeta['Authors'] = bookMeta['Authors'][0]
        else:
            bookMeta['Authors'] = bookMeta['Authors'][0]
    bookMeta['ISBN'] = bookMeta.pop('ISBN-13')
    title = bookMeta['Title']
    title = title.split(":")[0]
    bookMeta['Title'] = title
    auth = bookMeta['Authors']
    auth = auth.split(",")[0]
    auth = auth.split(";")[0]
    auth = auth.split("illustrated", 1)[0]
    auth = auth.split("and", 1)[0]
    auth = auth.split("with", 1)[0]
    bookMeta['Authors'] = auth
    pub = bookMeta['Publisher']
    pub = pub.split(" ")[0]
    bookMeta['Publisher'] = pub
    return bookMeta
def _doesexist(db, isbn):
    ext = Query()
    if db.search(ext.ISBN.matches(isbn)):
        return True
    else:
        return False
def _today():
    dat = datetime.date.today()
    return dat.strftime('%d/%m/%y')
_intro = Figlet(font = 'slant').renderText('xLibris')
############################################################
class xlibris(object):
    def __init__(self, directory = 'xlibris/'):
        self.directory = directory
        if not os.path.exists(self.directory):
            os.mkdir(self.directory)
    def new(self,dbname):
        '''Creates a new database assigns it to a current database instance to be used by all the other functions
            Usage:-
            new('database name') or new database_name
        '''
        self.db = TinyDB(self.directory + dbname + '.json')
        print 'New database {} created at {}'.format(dbname, self.directory + dbname + '.json' )
    def connect(self,name):
        '''Connect to an existing database for updating/Query
            Usage:-
            connect('name') or connect name
            where 'name' is the name of the existing database'''

        self.db = TinyDB(self.directory + name + '.json')
        print 'Connected to {}'.format(name)
    def display(self):
        try:
            print tabulate(_concat(self.db.all()), headers='keys', tablefmt="simple")
            print "\n"
            print self.count()
        except AttributeError:
            print '''No database Connected,
             to see a list of available databases use list_db
             or to make a new database use new'''

    def add(self, ISBN):
        '''Add books to the current working database
            Usage:-
            add(ISBN) or add ISBN'''
        if _doesexist(self.db, ISBN) == False:
            try:
                bookData = meta(ISBN)
                bookData = _cleanify(bookData)
                bookData['Date Added'] = _today()
                self.db.insert(bookData)
                print 'ISBN {} inserted'.format(ISBN)
            except:
                print 'ISBN {} not found. Please add details manually- '.format(ISBN)
                self.madd()
        else:
            print 'Book Already Exists'
    def madd(self):
        bookData = {}
        bookData['Authors'] = raw_input('Authors Name: ')
        bookData['ISBN'] = raw_input('ISBN: ')
        bookData['Language'] = raw_input('Language: ')
        bookData['Publisher'] = raw_input('Publisher: ')
        bookData['Title'] = raw_input('Title: ')
        bookData['Year'] = raw_input('Year: ')
        bookData['Date Added'] = _today()
        self.db.insert(bookData)
    def search(self, keyword):
        NewSearch = Query()
        title = self.db.search(NewSearch.Title == keyword)
        auth = self.db.search(NewSearch.Authors == keyword)
        pub = self.db.search(NewSearch.Publisher == keyword)
        isbn = self.db.search(NewSearch.ISBN == keyword)
        ttable = [title, auth, pub, isbn]

        for i in ttable:
            if i:
                print 'Matches Found for {} \n'.format(keyword)
                print tabulate(_concat(i), headers='keys', tablefmt="fancy_grid")
    def _blkadd(self, ISBNlist):
        with tqdm(ISBNlist) as pbar:
            for i in pbar:
                pbar.set_description("Adding %s "%i)
                self.add(i)
                pbar.update(1/len(ISBNlist)*100)
    def add_from_file(self, filename):
        with open(filename, 'rb') as f:
            raw = reader(f)
            final = list(raw)
        for i in range(len(final)):
            final[i] = str(final[i][0])
        self._blkadd(final)
        print 'Done'
    def change_title(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Title ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change title of :- {}'.format(title['Title'])
        self.db.update(change('Title'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def change_author(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Author ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change author of :- {}'.format(title['Title'])
        self.db.update(change('Authors'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def change_publisher(self, isbn):
        tmp = Query()
        def change(field):
            def transform(element):
                element[field] = raw_input('Enter Publisher ')
            return transform
        title = self.db.search(tmp.ISBN == isbn)[0]
        print 'Change Publisher of :- {}'.format(title['Title'])
        self.db.update(change('Publisher'), tmp.ISBN == isbn )
        print 'Entry Updated'
    def write_to_file(self, filename):
        try:
            data = tabulate(_concat(self.db.all()), headers='keys', tablefmt="simple")
        except AttributeError:
            print 'No database Connected'
        f = open('xlibris/' + filename + '.txt', 'w')
        f.write(data.encode('utf8'))
        f.write('\n'.encode('utf8'))
        f.write('--------------------'.encode('utf8'))
        f.write('\n'.encode('utf8'))
        f.write(self.count().encode('utf8'))
        f.close()
        print 'Written to {}'.format('xlibris/' + filename + '.txt')
    def purge_current(self):
        self.db.purge()
    def remove(self, isbn):
        tmp = Query()
        resp = raw_input('Delete \n {} \n ? (y/n)'.format(tabulate(self.db.search(tmp.ISBN == isbn), headers='keys')))
        resp = resp.lower()
        if resp == 'y':
            for i in ['Publisher', 'Title', 'Authors', 'Year', 'Date Added', 'Language', 'ISBN']:
                self.db.update(delete(i), tmp.ISBN == isbn)
            print 'Deleted'
        elif resp == 'n':
            print 'Spared'
    def lookup(self, keyword):
        data = _concat(self.db.all())
        title = data.pop('Title')
        auth = data.pop('Authors')
        choices = title + auth
        searchKey = process.extractBests(keyword, choices)
        for i in searchKey:
            if i[1] >= 90:
                self.search(i[0])
    def count(self):
        listisbn = _concat(self.db.all())
        listisbn = listisbn.pop('ISBN')
        return "Total {} books.".format(len(listisbn))
##############CONSOLE##############################
class xlibris_repl(cmd.Cmd):
    def __init__(self):
        cmd.Cmd.__init__(self)
        os.system('clear')
        self.prompt = 'xLibris>> '
        self.intro = _intro + '''A small library management system for your books written in python.
        Connect to a database and then just type the ISBN to add a book.
         Type help to find help
        Author - Shubham Bhushan
        '''
        self.current_db = xlibris()
        self.lst = ''
    def do_hist(self, args):
        '''Prints the list of commands entered recently'''
        print self._hist
    def do_exit(self, args):
        '''Exists from console after you type exit'''
        return -1
    def do_EOF(self, args):
        '''Exit on EOF character'''
        return True
    def do_list_db(self, args):
        '''Lists the databases in the native folder.
        Databases are stored as .json files
        simply type list_db to view available databases'''
        # listdb += [i for i in os.listdir('xlibris/') if i.endswith('.json')]
        listdb = {}
        dbs = []
        for i in os.listdir('xlibris/'):
            if i.endswith('.json'):
                dbs.append(i)
        listdb['Databases'] = dbs
        print tabulate(listdb, headers='keys', tablefmt='fancy_grid')
    def do_help(self, args):
        '''Get help on commands.
        type help for a list of available commands
        or help <command> for instructions on a specific command'''
        cmd.Cmd.do_help(self, args)
    def preloop(self):
        cmd.Cmd.preloop(self)
        self._hist    = []
        self._locals  = {}
        self._globals = {}
    def postloop(self):
        cmd.Cmd.postloop(self)
        print 'Exiting xLibris... '
    def precmd(self, args):
        self._hist += [args.split()]
        return args
    def do_new(self, args):
        '''Creates a new database.
        new <name> where <name> is the name of db you wish to create.'''
        self.current_db.new(args)
    def do_connect(self, args):
        '''Connect to an existing database
        connect <name of database without .json>'''
        self.current_db.connect(args)
    def do_add(self, args):
        '''Adds books to the current database.
        add <isbn> to add a book.'''
        self.current_db.add(args)
    def default(self, args):
        '''Adds the book to current database when just ISBN is typed'''
        try:
            self.current_db.add(str(int(args)))
        except ValueError:
            print "Only ISBN, to see a list of functions type help"
    def do_display(self, args):
        '''Prints the current database to screen'''
        self.current_db.display()
    def do_search(self, args):
        '''Searches the database for given keyword.
        Usage
        search <keyword>
        legal keywords include ISBN, author, Title. '''
        self.current_db.search(args)
    def do_lookup(self, args):
        '''Fuzzy Searching can search from partial titles or Author Names
        lookup <keyword>'''
        self.current_db.lookup(args)
    def do_addFromFile(self, args):
        '''Add books from a csv file containing ISBNs.
        type add_from_file <path/filename.csv>'''
        self.current_db.add_from_file(args)
    def do_writeToFile(self, args):
        '''write the current database in a formatted table to a file on the disk
        writeToFile <filename>'''
        self.current_db.write_to_file(args)
    def do_purgeCurrent(self, args):
        '''Purges current database. Beware.
        type purgeCurrent'''
        self.current_db.purge_current()
    def do_changeTitle(self, args):
        '''Helps you change title of the book you have added.
        changeTitle ISBN to change the title of the book'''
        try:
            self.current_db.change_title(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def do_changeAuthor(self, args):
        '''Helps you change the Author of the book you have added.
        changeAuthor ISBN to change the author of the book'''
        try:
            self.current_db.change_author(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def do_changePublisher(self, args):
        '''Helps you change the Publisher of the book you have added.
        changeAuthor ISBN to change the author of the book'''
        try:
            self.current_db.change_publisher(args)
        except IndexError:
            print 'You have to enter the correct ISBN too'
    def emptyline(self):
        print '''Create a new database using new or connect to existing database using connect
        or type help to see a list of available commands. You can connect to one of the following
        '''
        self.do_list_db('foo')
    def do_shell(self, args):
        '''Use shell commands by affixing them with ! sign'''
        os.system(args)
    def do_remove(self, args):
        '''Remove a book from the database.
        remove <isbn>
        * ALthough I have no idea why anyone would use this heinous function'''
        self.current_db.remove(args)
#TODO - Add remove function
if __name__ == '__main__':
    xlibris_repl().cmdloop()
<|endoftext|>


<commit_before># Copyright (c) The SimpleFIN Team
# See LICENSE for details.

from twisted.python.filepath import FilePath
import argparse

root = FilePath(__file__).parent()

parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(help='sub-command help')

def serve(args):
    import sys
    from twisted.python import log
    from siloscript.server import UIServer
    from siloscript.storage import MemoryStore

    log.startLogging(sys.stdout)
    store = MemoryStore()
    server = UIServer(store, args.scripts, static_root=args.static)
    server.app.run('0.0.0.0', args.port)

server_parser = subparsers.add_parser('serve', help='Start HTTP server')
server_parser.add_argument('--port', '-p',
    type=int,
    default=9600,
    help='Port to serve HTTP server on. (default: %(default)s)')
server_parser.add_argument('--scripts', '-s',
    default=root.child('data').child('scripts').path,
    help='Path to executable scripts.  (default: %(default)s)')
server_parser.add_argument('--static-root', '-S',
    default=root.child('data').child('static'),
    help='Path to static files served at /static.  (default: %(default)s)')
server_parser.set_defaults(func=serve)

def run():
    args = parser.parse_args()
    args.func(args)
<commit_msg>Fix typo<commit_after># Copyright (c) The SimpleFIN Team
# See LICENSE for details.

from twisted.python.filepath import FilePath
import argparse

root = FilePath(__file__).parent()

parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(help='sub-command help')

def serve(args):
    import sys
    from twisted.python import log
    from siloscript.server import UIServer
    from siloscript.storage import MemoryStore

    log.startLogging(sys.stdout)
    store = MemoryStore()
    server = UIServer(store, args.scripts, static_root=args.static_root)
    server.app.run('0.0.0.0', args.port)

server_parser = subparsers.add_parser('serve', help='Start HTTP server')
server_parser.add_argument('--port', '-p',
    type=int,
    default=9600,
    help='Port to serve HTTP server on. (default: %(default)s)')
server_parser.add_argument('--scripts', '-s',
    default=root.child('data').child('scripts').path,
    help='Path to executable scripts.  (default: %(default)s)')
server_parser.add_argument('--static-root', '-S',
    default=root.child('data').child('static'),
    help='Path to static files served at /static.  (default: %(default)s)')
server_parser.set_defaults(func=serve)

def run():
    args = parser.parse_args()
    args.func(args)
<|endoftext|>