<commit_before>
import collections
import copy
import functools
import re
import string
import struct

from .spec import Spec, tokenize_lines, Scalar, atom_to_spec, isconverter
from .util import OrderedDict, ConvertError

class NamedTuple(Spec):
    @property
    def width(self):
        return sum(s.width for s in self._pos_specs)

    def __init__(self, key_map=()):
        from .mapping import _norm_mapping_key_spec
        self._key_map, cm = _norm_mapping_key_spec(key_map)
        self._field_names = [n for n, c in self._key_map]

        self._pos_converts = []
        has_pos_spec = False
        for name, spec in self._key_map:
            if isconverter(spec):
                self._pos_converts.append(spec)
                has_pos_spec = True
            else:
                self._pos_converts.append(_null_convert)
        if not has_pos_spec:
            self._pos_converts = None

        nt_class = collections.namedtuple('BaseNamedTuple', self._field_names)
        self._my_class = type('NamedTuple', (_NamedTupleValue, nt_class), {})

        # We inspect the positional specs in the key map to determine which
        # ones have their own parsing routine. If they do not, then we treat
        # them as a simple string and strip the trailing white space
        self._sub_unpacks = []
        for idx, (name, spec)  in enumerate(self._key_map):
            if hasattr(spec, 'unpack'):
                self._sub_unpacks.append(spec.unpack)
            else:
                self._sub_unpacks.append(string.rstrip)

        self._pack_funcs = []
        for name, spec in self._key_map:
            if hasattr(spec, 'pack'):
                func = spec.pack
            elif hasattr(spec, 'width'):
                func = functools.partial(_pack_string, spec.width)
            else:
                func = functools.partial(_pack_string, spec)
            self._pack_funcs.append(func)

        # Use the built-in structs module to do the actual string split in C
        self._struct = struct.Struct(self._struct_fmt)

    def unpack(self, text_line):
        try:
            values = self._struct.unpack_from(text_line)
        except struct.error:
            # pad the line out so that struct will take it
            values = self._struct.unpack_from(text_line.ljust(self.width))

        values = [s(v) for s, v in zip(self._sub_unpacks, values)]
        return self._my_class(values, self)

    def pack(self, listval):
        return ''.join(s(v) for s, v in zip(self._pack_funcs, listval))

    def from_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            try:
                nvalue.append(convert.from_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return self._my_class(nvalue, self, convert_errors)

    def to_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            try:
                nvalue.append(convert.to_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return self._my_class(nvalue, self, convert_errors)

    @property
    def _struct_fmt(self):
        return ''.join('%ds' % f.width for name, f in self._key_map)

class _NamedTupleValue(object):
    def __new__(cls, value, spec, convert_errors=None):
        i = cls.__bases__[1].__new__(cls, *value)
        i._spec = spec
        i._convert_errors = convert_errors or []
        return i

    def from_text(self):
        return self._spec.from_text(self)

    def to_text(self):
        return self._spec.to_text(self)

    def pack(self):
        return self._spec.pack(self)

class _BaseSequence(Spec):
    _factory = None

    def __init__(self, pos_specs):
        self._pos_specs = _norm_sequence_pos_specs(pos_specs)
        # Use the built-in structs module to do the actual string split in C
        self._struct = struct.Struct(self._struct_fmt)

        self._setup_unpack_funs()
        self._setup_pack_funs()
        self._setup_convert_objs()

    @property
    def width(self):
        return sum(s.width for s in self._pos_specs)

    ## Parsing
    def unpack(self, string):
        try:
            values = self._struct.unpack_from(string)
        except struct.error:
            # pad the line out so that struct will take it
            values = self._struct.unpack_from(string.ljust(self.width))
        v = [s(v) for s, v in zip(self._sub_unpacks, values)]
        return self._factory(v, self)

    @property
    def _struct_fmt(self):
        return ''.join('%ds' % f.width for  f in self._pos_specs)

    def _setup_unpack_funs(self):
        """ Assign a list of parsing functions to the state of the spec type
        to allow fast iteration of field unpackrs during parsing. If the
        positional spec has a unpack method, use that. Otherwise, we just
        strip whitespace off the right."""

        self._sub_unpacks = []
        for idx, pos_spec  in enumerate(self._pos_specs):
            if hasattr(pos_spec, 'unpack'):
                self._sub_unpacks.append(pos_spec.unpack)
            else:
                self._sub_unpacks.append(string.rstrip)

    ## Formatting
    def pack(self, listval):
        return ''.join(s(v) for s, v in zip(self._pack_funcs, listval))

    def _setup_pack_funs(self):
        self._pack_funcs = []
        for pos_spec in self._pos_specs:
            if hasattr(pos_spec, 'pack'):
                func = pos_spec.pack
            elif hasattr(pos_spec, 'width'):
                func = functools.partial(_pack_string, pos_spec.width)
            else:
                func = functools.partial(_pack_string, pos_spec)
            self._pack_funcs.append(func)

    ## Conversion Responsibilities
    def from_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            try:
                nvalue.append(convert.from_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return self._my_class(nvalue, self, convert_errors)

    def to_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            try:
                nvalue.append(convert.to_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return self._my_class(nvalue, self, convert_errors)

    def _setup_convert_objs(self):
        self._pos_converts = []
        has_pos_convert = False
        for spec in self._pos_specs:
            if isconverter(spec):
                self._pos_converts.append(spec)
                has_pos_convert = True
            else:
                self._pos_converts.append(_null_convert)
        # Save the from/to text loops from iterating
        if not has_pos_convert:
            self._pos_converts = None

class _NullConvert(object):
    """ A Null converter to keep from having a lot of if statements in
    to_text/from_text methods """
    to_text = lambda s, v: v
    from_text = lambda s, v: v
_null_convert = _NullConvert()

def _norm_sequence_pos_specs(specs):
    """ Normalize sequence specs passed in from client calls. These can have
    ints, strings, and types in them.
    """
    if isinstance(specs, basestring):
        specs = tokenize_lines(specs)
    return map(atom_to_spec, specs)

class List(_BaseSequence):
    def __init__(self, *a, **k):
        _BaseSequence.__init__(self, *a, **k)
        self._factory = _ListValue

    ## Conversion Responsibilities
    def from_text(self, avalue):
        nvalue = copy.deepcopy(avalue)
        if not self._pos_converts:
            return nvalue

        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            if convert:
                try:
                    nvalue[idx] = convert.from_text(value)
                except ConvertError, e:
                    convert_errors.append((idx, value, e))
        return nvalue

    def to_text(self, avalue):
        nvalue = copy.deepcopy(avalue)
        if not self._pos_converts:
            return nvalue

        for idx, (value, convert) in enumerate(zip(avalue, self._pos_converts)):
            if convert:
                try:
                    nvalue[idx] = convert.to_text(value)
                except ConvertError, e:
                    convert_errors.append((idx, value, e))
        return nvalue

class _ListValue(list):
    def __init__(self, other, spec):
        list.__init__(self, other)
        self._spec = spec
        self._convert_errors = []

    def __copy__(self):
        rec = _ListValue(self, self._spec)
        rec._convert_errors = list(self._convert_errors)
        return rec

    def __deepcopy__(self, memo):
        rec = _ListValue(self, self._spec)
        rec._convert_errors = copy.deepcopy(self._convert_errors, memo)
        return rec

    def add_convert_error(self, index, value, error):
        self._convert_errors.append((index, value, error))

    def convert_errors(self):
        return list(self._convert_errors)

    ## Conversion Responsibilities
    def to_text(self):
        return self._spec.to_text(self)

    def from_text(self):
        return self._spec.from_text(self)

    def pack(self):
        return self._spec.pack(self)

## Tuple
class Tuple(_BaseSequence):
    def __init__(self, *a, **k):
        _BaseSequence.__init__(self, *a, **k)
        self._factory = _TupleValue

    def from_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, value in enumerate(avalue):
            try:
                conv = self._pos_converts[idx]
            except IndexError:
                nvalue.append(value)
                continue
            try:
                nvalue.append(conv.from_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return _TupleValue(nvalue, self, convert_errors)

    def to_text(self, avalue):
        if not self._pos_converts:
            return copy.copy(avalue)

        convert_errors = []
        nvalue = []
        for idx, value in enumerate(avalue):
            try:
                conv = self._pos_converts[idx]
            except IndexError:
                nvalue.append(value)
                continue
            try:
                nvalue.append(conv.to_text(value))
            except ConvertError, e:
                nvalue.append(value)
                convert_errors.append((idx, value, e))
        return _TupleValue(nvalue, self, convert_errors)

class _TupleValue(tuple):
    def __new__(cls, other, spec, convert_errors=None):
        inst = tuple.__new__(cls, other)
        inst._spec = spec
        inst._convert_errors = convert_errors or []
        return inst

    def __init__(self, other, spec, convert_errors=None):
        tuple.__init__(self, other)

    def __copy__(self):
        rec = _TupleValue(self, self._spec)
        rec._convert_errors = list(self._convert_errors)
        return rec

    def __deepcopy__(self, memo):
        rec = _TupleValue(self, self._spec)
        rec._convert_errors = copy.deepcopy(self._convert_errors, memo)
        return rec

    def convert_errors(self):
        return list(self._convert_errors)

    def from_text(self):
        return self._spec.from_text(self)

    def to_text(self):
        return self._spec.to_text(self)

    def pack(self):
        return self._spec.pack(self)

class Array(Spec):
    def __init__(self, count, spec):
        self._count = count
        self._spec = atom_to_spec(spec)

        # Do this lookup here to save it from doing in unpack
        self._sub_unpack = hasattr(spec, 'unpack')
        if isconverter(spec):
            self._converter = spec
        else:
            self._converter = None

    @property
    def width(self):
        return self._count * self._spec.width

    ## Layout Responsibilities
    def unpack(self, text_line):
        if len(text_line) < self.width:
            # pad the line out so that struct won't blow up
            text_line += ' ' * (self.width - len(text_line))
        parts = map(''.join, zip(*[iter(text_line)] * self._spec.width))
        if self._sub_unpack:
            parts = map(self._spec.unpack, parts)
        return ArrayValue(parts, self)

    def pack(self, avalue):
        parts = []
        field = self._spec
        for value in avalue:
            if hasattr(field, 'pack'):
                value = field.pack(value)
            if len(value) > field.width:
                value = value[:field.width]
            elif len(value) < field.width:
                value = value.ljust(width)
            parts.append(value)
        return ''.join(parts)

    ## Converter Responsibilities
    def from_text(self, avalue):
        nvalue = copy.deepcopy(avalue)
        if self._converter:
            for idx, value in enumerate(avalue):
                try:
                    nvalue[idx] = self._converter.from_text(value)
                except ConvertError, e:
                    nrec.add_convert_error(idx, value, e)
        return nvalue

    def to_text(self, avalue):
        nvalue = copy.deepcopy(avalue)
        if self._converter:
            for idx, value in enumerate(avalue):
                try:
                    nvalue[idx] = self._converter.to_text(value)
                except ConvertError, e:
                    nrec.add_convert_error(idx, value, e)
        return nvalue

class ArrayValue(list):
    def __init__(self, values, field):
        list.__init__(self, values)
        self._field = field
        self._convert_errors = []

    def __copy__(self):
        rec = ArrayValue(self, self._field)
        rec._convert_errors = self._convert_errors
        return rec

    def __deepcopy__(self, memo):
        rec = ArrayValue(self, self._field)
        rec._convert_errors = copy.deepcopy(self._convert_errors, memo)
        return rec

    def add_convert_error(self, index, value, error):
        self._convert_errors.append((index, value, error))

    def convert_errors(self):
        return list(self._convert_errors)

def _pack_string(length, value):
    return value[:length].ljust(length)
<commit_msg>massive rewrite. Added NamedTuple to the BaseSequence class. removed the from_text/to_text interface on the containers. removed all error lists on value objects.<commit_after>
import collections
import copy
import functools
import re
import string
import struct

from .spec import Spec, atom_to_spec_seq, atom_to_scalar, atom_to_spec_map

from .util import OrderedDict, ConvertError

class BaseSequence(Spec):
    _itype = None
    _str_itype = None

    def __init__(self, pos_specs=()):
        self._pos_specs = atom_to_spec_seq(pos_specs)
        self._setup_from_str_funs()
        self._setup_to_str_funs()
        self._unpack_funs = [p.unpack for p in self._pos_specs]
        self._pack_funs = [p.pack for p in self._pos_specs]
        self._struct = struct.Struct(self._struct_fmt)

    @property
    def width(self):
        return sum(s.width for s in self._pos_specs)

    ## unpack
    def unpack(self, text_line):
        try:
            values = self._struct.unpack_from(text_line)
        except struct.error:
            # pad the line out so that struct will take it
            values = self._struct.unpack_from(text_line.ljust(self.width))

        values = [s(v) for s, v in zip(self._unpack_funs, values)]
        for idx, from_str in self._from_str_funs:
            values[idx] = from_str(values[idx])
        return self._itype(values, self)

    def _setup_from_str_funs(self):
        # Functions to call when we convert from a string to a value
        self._from_str_funs = []
        for idx, spec in enumerate(self._pos_specs):
            if hasattr(spec, 'from_text'):
                self._from_str_funs.append((idx, spec.from_text))

    @property
    def _struct_fmt(self):
        return ''.join('%ds' % s.width for s in self._pos_specs)

    ## Pack
    def pack(self, value):
        # Shortcut
        if not self._to_str_funs:
            return ''.join(s(v) for s, v in zip(self._pack_funs, value))

        str_values = list(value)
        for idx, to_str in self._to_str_funs:
            str_values[idx] = to_str(value[idx])
        return ''.join(s(v) for s, v in zip(self._pack_funs, str_values))

    def as_strings(self, value):
        """ Return a builtin NamedTuple that is an 'export' of the value
        into strings
        """
        if not self._to_str_funs:
            return self._str_itype(value)

        str_value = list(value)
        for idx, to_str in self._to_str_funs:
            str_value[idx] = to_str(value[idx])
        return self._str_itype(*str_value)

    def _setup_to_str_funs(self):
        # Functions to call when we convert from a value to a string
        self._to_str_funs = []
        for idx, spec in enumerate(self._pos_specs):
            if hasattr(spec, 'to_text'):
                self._to_str_funs.append((idx, spec.to_text))

## NamedTuple
class NamedTuple(BaseSequence):
    def __init__(self, key_map=()):
        self._key_map = atom_to_spec_map(key_map)
        self._field_names = [n for n, c in self._key_map]
        self._str_itype = collections.namedtuple('BaseNamedTuple', self._field_names)
        self._itype = type('NamedTupleValue', (_NamedTupleValue, self._str_itype), {})

        pos_specs = [c for n, c in self._key_map]
        BaseSequence.__init__(self, pos_specs)

class _NamedTupleValue(object):
    def __new__(cls, value, spec):
        i = cls.__bases__[1].__new__(cls, *value)
        i._spec = spec
        return i

    def pack(self):
        return self._spec.pack(self)

    def as_strings(self):
        return self._spec.as_strings(self)

## Tuple
class Tuple(BaseSequence):
    def __init__(self, *a, **k):
        self._itype = _TupleValue
        self._str_itype = tuple
        BaseSequence.__init__(self, *a, **k)

class _TupleValue(tuple):
    def __new__(cls, other, spec, convert_errors=None):
        inst = tuple.__new__(cls, other)
        inst._spec = spec
        inst._convert_errors = convert_errors or []
        return inst

    def __init__(self, other, spec, convert_errors=None):
        tuple.__init__(self, other)

    def __copy__(self):
        rec = _TupleValue(self, self._spec)
        rec._convert_errors = list(self._convert_errors)
        return rec

    def __deepcopy__(self, memo):
        rec = _TupleValue(self, self._spec)
        rec._convert_errors = copy.deepcopy(self._convert_errors, memo)
        return rec

    def convert_errors(self):
        return list(self._convert_errors)

    def pack(self):
        return self._spec.pack(self)

## List
class List(BaseSequence):
    def __init__(self, *a, **k):
        self._str_itype = list
        self._itype = _ListValue
        BaseSequence.__init__(self, *a, **k)

class _ListValue(list):
    def __init__(self, other, spec):
        list.__init__(self, other)
        self._spec = spec

    def __copy__(self):
        rec = type(self)(self, self._spec)
        return rec

    def __deepcopy__(self, memo):
        rec = type(self)(self, self._spec)
        return rec

    ## List Protocol
    def __setitem__(self, index, str_value):
        if not isinstance(str_value, basestring):
            return list.__setitem__(self, index, str_value)

        if isinstance(index, slice):
            indexes = range(*index.indices(len(self)))
        else:
            indexes = [index]

        for index in indexes:
            for fun_index, fun in self._spec._from_str_funs:
                if fun_index == index:
                    value = fun(str_value)
                    break
            else:
                value = str_value
            list.__setitem__(self, index, value)

    def __delitem__(self, index):
        raise TypeError("stype lists cannot change size")

    def append(self, value):
        raise TypeError("stype lists cannot change size")

    def extend(self, other):
        raise TypeError("stype lists cannot change size")

    def insert(self, index, value):
        raise TypeError("stype lists cannot change size")

    def pop(self, index=-1):
        raise TypeError("stype lists cannot change size")

    def remove(self, index, value):
        raise TypeError("stype lists cannot change size")

    def reverse(self):
        raise TypeError("Operation not supported")

    ## stypes Protocol
    def pack(self):
        return self._spec.pack(self)

## Array
class Array(Spec):
    """ Homogenous list """
    def __init__(self, count, spec):
        self._count = count
        self._spec = atom_to_scalar(spec)
        self._struct = struct.Struct(self._struct_fmt)

        if hasattr(self._spec, 'from_text'):
            self._from_str_fun = self._spec.from_text
        else:
            self._from_str_fun = None

        if hasattr(self._spec, 'to_text'):
            self._to_str_fun = self._spec.to_text
        else:
            self._to_str_fun = None

    @property
    def width(self):
        return self._count * self._spec.width

    @property
    def _struct_fmt(self):
        return ('%ds' % self._spec.width)  * self._count

    def unpack(self, text_line):
        try:
            values = self._struct.unpack_from(text_line)
        except struct.error:
            # pad the line out so that struct will take it
            values = self._struct.unpack_from(text_line.ljust(self.width))

        values = map(self._spec.unpack, values)
        if self._from_str_fun:
            values = map(self._from_str_fun, values)
        return _ListValue(values, self)

    def pack(self, values):
        if self._to_str_fun:
            values = map(self._to_str_fun, values)
        return ''.join(map(self._spec.pack, values))

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014–2016 Molly White
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import tweepy
from secrets import *
from random import choice
import os

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))

tweets = [
  "Please ask for help if you need it.\n\nNational Suicide Prevention Lifeline: 1-800-273-8255",
  "National Suicide Prevention Lifeline: 1-800-273-8255\n\nCrisis Text Line: Text START to 741-741\n\nInternational Support: http://www.suicide.org/international-suicide-hotlines.html",
  "If you or someone you know needs help, or just needs to talk, please call the National Suicide Prevention Lifeline at 1-800-273-8255",
  "If you or someone you know is thinking about suicide, or dealing with a suicide loss, call the National Suicide Prevention Lifeline at 1-800-273-8255"
]

if __name__ == "__main__":
    """Tweet one of several suicide hotline messages."""
    auth = tweepy.OAuthHandler(C_KEY, C_SECRET)
    auth.set_access_token(A_TOKEN, A_TOKEN_SECRET)
    api = tweepy.API(auth)

    api.update_status(choice(tweets))
<commit_msg>No way to have cron do this kinda-randomly, so<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014–2016 Molly White
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import tweepy
from secrets import *
import random
import os

__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))

tweets = [
  "Please ask for help if you need it.\n\nNational Suicide Prevention Lifeline: 1-800-273-8255",
  "National Suicide Prevention Lifeline: 1-800-273-8255\n\nCrisis Text Line: Text START to 741-741\n\nInternational Support: http://www.suicide.org/international-suicide-hotlines.html",
  "If you or someone you know needs help, or just needs to talk, please call the National Suicide Prevention Lifeline at 1-800-273-8255",
  "If you or someone you know is thinking about suicide, or dealing with a suicide loss, call the National Suicide Prevention Lifeline at 1-800-273-8255"
]

if __name__ == "__main__":
    """Tweet one of several suicide hotline messages."""
    if random.randint(0, 5) == 0:
      auth = tweepy.OAuthHandler(C_KEY, C_SECRET)
      auth.set_access_token(A_TOKEN, A_TOKEN_SECRET)
      api = tweepy.API(auth)

      api.update_status(random.choice(tweets))
<|endoftext|>


<commit_before>'''
Implements the targetcli backstores related UI.

This file is part of targetcli.
Copyright (c) 2011 by RisingTide Systems LLC

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, version 3 (AGPLv3).

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

from ui_node import UINode, UIRTSLibNode
from rtslib import RTSRoot
from rtslib import FileIOBackstore, BlockBackstore
from rtslib import PSCSIBackstore, RDMCPBackstore
from rtslib import FileIOStorageObject, BlockStorageObject
from rtslib import PSCSIStorageObject, RDMCPStorageObject
from rtslib.utils import (get_block_type, is_disk_partition)
from configshell import ExecutionError
import os
import re

def human_to_bytes(hsize, kilo=1024):
    '''
    This function converts human-readable amounts of bytes to bytes.
    It understands the following units :
        - I{B} or no unit present for Bytes
        - I{k}, I{K}, I{kB}, I{KB} for kB (kilobytes)
        - I{m}, I{M}, I{mB}, I{MB} for MB (megabytes)
        - I{g}, I{G}, I{gB}, I{GB} for GB (gigabytes)
        - I{t}, I{T}, I{tB}, I{TB} for TB (terabytes)

    Note: The definition of I{kilo} defaults to 1kB = 1024Bytes.
    Strictly speaking, those should not be called I{kB} but I{kiB}.
    You can override that with the optional kilo parameter.

    @param hsize: The human-readable version of the Bytes amount to convert
    @type hsize: string or int
    @param kilo: Optional base for the kilo prefix
    @type kilo: int
    @return: An int representing the human-readable string converted to bytes
    '''
    size = str(hsize).replace("g","G").replace("K","k")
    size = size.replace("m","M").replace("t","T")
    if not re.match("^[0-9]+[T|G|M|k]?[B]?$", size):
        raise RTSLibError("Cannot interpret size, wrong format: %s" % hsize)

    size = size.rstrip('B')

    units = ['k', 'M', 'G', 'T']
    try:
        power = units.index(size[-1]) + 1
    except ValueError:
        power = 0
        size = int(size)
    else:
        size = int(size[:-1])

    return size * (int(kilo) ** power)

def bytes_to_human(size, kilo=1024.0):
    if not size:
        return ""
    for x in ['bytes','K','M','G','T']:
        if size < kilo:
            return "(%3.1f%s) " % (size, x)
        size /= kilo

class UIBackstores(UINode):
    '''
    The backstores container UI.
    '''
    def __init__(self, parent):
        UINode.__init__(self, 'backstores', parent)
        self.cfs_cwd = "%s/core" % self.cfs_cwd
        self.refresh()

    def refresh(self):
        self._children = set([])
        UIPSCSIBackstore(self)
        UIRDMCPBackstore(self)
        UIFileIOBackstore(self)
        UIBlockBackstore(self)

class UIBackstore(UINode):
    '''
    A backstore UI.
    '''
    def __init__(self, plugin, parent):
        UINode.__init__(self, plugin, parent)
        self.cfs_cwd = "%s/core" % self.cfs_cwd
        self.refresh()

    def refresh(self):
        self._children = set([])
        for so in RTSRoot().storage_objects:
            if so.backstore.plugin == self.name:
                ui_so = UIStorageObject(so, self)
                ui_so.name = so.name

    def summary(self):
        no_storage_objects = len(self._children)
        if no_storage_objects > 1:
            msg = "%d Storage Objects" % no_storage_objects
        else:
            msg = "%d Storage Object" % no_storage_objects
        return (msg, None)

    def prm_buffered(self, buffered):
        buffered = \
                self.ui_eval_param(buffered, 'bool', True)
        if buffered:
            self.shell.log.info("Using buffered mode.")
        else:
            self.shell.log.info("Not using buffered mode.")
        return buffered

    def ui_command_delete(self, name):
        '''
        Recursively deletes the storage object having the specified I{name}. If
        there are LUNs using this storage object, they will be deleted too.

        EXAMPLE
        =======
        B{delete mystorage}
        -------------------
        Deletes the storage object named mystorage, and all associated LUNs.
        '''
        self.assert_root()
        try:
            child = self.get_child(name)
        except ValueError:
            self.shell.log.error("No storage object named %s." % name)
        else:
            hba = child.rtsnode.backstore
            child.rtsnode.delete()
            if not list(hba.storage_objects):
                hba.delete()
            self.remove_child(child)
            self.shell.log.info("Deleted storage object %s." % name)
            self.parent.parent.refresh()

    def ui_complete_delete(self, parameters, text, current_param):
        '''
        Parameter auto-completion method for user command delete.
        @param parameters: Parameters on the command line.
        @type parameters: dict
        @param text: Current text of parameter being typed by the user.
        @type text: str
        @param current_param: Name of parameter to complete.
        @type current_param: str
        @return: Possible completions
        @rtype: list of str
        '''
        if current_param == 'name':
            names = [child.name for child in self.children]
            completions = [name for name in names
                           if name.startswith(text)]
        else:
            completions = []

        if len(completions) == 1:
            return [completions[0] + ' ']
        else:
            return completions

    def next_hba_index(self):
        self.shell.log.debug("%r" % [(backstore.plugin, backstore.index)
                                     for backstore in RTSRoot().backstores])
        indexes = [backstore.index for backstore in RTSRoot().backstores
                   if backstore.plugin == self.name]
        self.shell.log.debug("Existing %s backstore indexes: %r"
                             % (self.name, indexes))
        for index in range(1048576):
            if index not in indexes:
                backstore_index = index
                break

        if backstore_index is None:
            raise ExecutionError("Cannot find an available backstore index.")
        else:
            self.shell.log.debug("First available %s backstore index is %d."
                                 % (self.name, backstore_index))
            return backstore_index

    def assert_available_so_name(self, name):
        names = [child.name for child in self.children]
        if name in names:
            raise ExecutionError("Storage object %s/%s already exist."
                                 % (self.name, name))

class UIPSCSIBackstore(UIBackstore):
    '''
    PSCSI backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'pscsi', parent)

    def ui_command_create(self, name, dev):
        '''
        Creates a PSCSI storage object, with supplied name and SCSI device. The
        SCSI device I{dev} can either be a path name to the device, in which
        case it is recommended to use the /dev/disk/by-id hierarchy to have
        consistent naming should your physical SCSI system be modified, or an
        SCSI device ID in the H:C:T:L format, which is not recommended as SCSI
        IDs may vary in time.
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = PSCSIBackstore(self.next_hba_index(), mode='create')

        if get_block_type(dev) is not None or is_disk_partition(dev):
            self.shell.log.info("Note: block backstore recommended for "
                                "SCSI block devices")

        try:
            so = PSCSIStorageObject(backstore, name, dev)
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created pscsi storage object %s using %s"
                            % (name, dev))
        return self.new_node(ui_so)

class UIRDMCPBackstore(UIBackstore):
    '''
    RDMCP backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'ramdisk', parent)

    def ui_command_create(self, name, size):
        '''
        Creates an RDMCP storage object. I{size} is the size of the ramdisk.

        SIZE SYNTAX
        ===========
        - If size is an int, it represents a number of bytes.
        - If size is a string, the following units can be used:
            - B{B} or no unit present for bytes
            - B{k}, B{K}, B{kB}, B{KB} for kB (kilobytes)
            - B{m}, B{M}, B{mB}, B{MB} for MB (megabytes)
            - B{g}, B{G}, B{gB}, B{GB} for GB (gigabytes)
            - B{t}, B{T}, B{tB}, B{TB} for TB (terabytes)
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = RDMCPBackstore(self.next_hba_index(), mode='create')
        try:
            so = RDMCPStorageObject(backstore, name, human_to_bytes(size))
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created ramdisk %s with size %s."
                            % (name, size))
        return self.new_node(ui_so)

class UIFileIOBackstore(UIBackstore):
    '''
    FileIO backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'fileio', parent)

    def _create_file(self, filename, size, sparse=True):
        f = open(filename, "w+")
        try:
            if sparse:
                f.seek(size-1)
                f.write("\0")
            else:
                self.shell.log.info("Writing %s bytes" % size)
                while size > 0:
                    write_size = min(size, 1024)
                    f.write("\0" * write_size)
                    size -= write_size
        except IOError:
            f.close()
            os.remove(filename)
            raise ExecutionError("Could not expand file to size")
        f.close()

    def ui_command_create(self, name, file_or_dev, size=None, buffered=None,
                          sparse=None):
        '''
        Creates a FileIO storage object. If I{file_or_dev} is a path to a
        regular file to be used as backend, then the I{size} parameter is
        mandatory. Else, if I{file_or_dev} is a path to a block device, the
        size parameter B{must} be ommited. If present, I{size} is the size of
        the file to be used, I{file} the path to the file or I{dev} the path to
        a block device. The I{buffered} parameter is a boolean stating
        whether or not to enable buffered mode. It is enabled by default
        (asynchronous mode). The I{sparse} parameter is only applicable when
        creating a new backing file. It is a boolean stating if the
        created file should be created as a sparse file (the default), or
        fully initialized.

        SIZE SYNTAX
        ===========
        - If size is an int, it represents a number of bytes.
        - If size is a string, the following units can be used:
            - B{B} or no unit present for bytes
            - B{k}, B{K}, B{kB}, B{KB} for kB (kilobytes)
            - B{m}, B{M}, B{mB}, B{MB} for MB (megabytes)
            - B{g}, B{G}, B{gB}, B{GB} for GB (gigabytes)
            - B{t}, B{T}, B{tB}, B{TB} for TB (terabytes)
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        self.shell.log.debug("Using params size=%s buffered=%s sparse=%s"
                             % (size, buffered, sparse))

        sparse = self.ui_eval_param(sparse, 'bool', True)

        backstore = FileIOBackstore(self.next_hba_index(), mode='create')

        is_dev = get_block_type(file_or_dev) is not None \
                or is_disk_partition(file_or_dev)

        if is_dev:
            if size:
                self.shell.log.info("Block device, size parameter ignored")
                size = None
            self.shell.log.info("Note: block backstore preferred for best results")
        else:
            # use given file size only if backing file does not exist
            if os.path.isfile(file_or_dev):
                new_size = os.path.getsize(file_or_dev)
                if size:
                    self.shell.log.info("%s exists, using its size (%s bytes) instead"
                                        % (file_or_dev, new_size))
                size = new_size
            else:
                # create file and extend to given file size
                if not size:
                    raise ExecutionError("Attempting to create file for new" +
                                         " fileio backstore, need a size")
                size = human_to_bytes(size)
                self._create_file(file_or_dev, size, sparse)

        try:
            so = FileIOStorageObject(
                backstore, name, file_or_dev,
                size,
                buffered_mode=self.prm_buffered(buffered))
        except:
            backstore.delete()
            raise

        self.shell.log.info("Created fileio %s with size %s"
                            % (name, size))
        ui_so = UIStorageObject(so, self)

        return self.new_node(ui_so)

class UIBlockBackstore(UIBackstore):
    '''
    Block backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'block', parent)

    def ui_command_create(self, name, dev):
        '''
        Creates an Block Storage object. I{dev} is the path to the TYPE_DISK
        block device to use.
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = BlockBackstore(self.next_hba_index(), mode='create')
        try:
            so = BlockStorageObject(backstore, name, dev)
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created block storage object %s using %s."
                            % (name, dev))
        return self.new_node(ui_so)

class UIStorageObject(UIRTSLibNode):
    '''
    A storage object UI.
    '''
    def __init__(self, storage_object, parent):
        name = storage_object.name
        UIRTSLibNode.__init__(self, name, storage_object, parent)
        self.cfs_cwd = storage_object.path
        self.refresh()

    def ui_command_version(self):
        '''
        Displays the version of the current backstore's plugin.
        '''
        backstore = self.rtsnode.backstore
        self.shell.con.display("Backstore plugin %s %s"
                               % (backstore.plugin, backstore.version))

    def summary(self):
        so = self.rtsnode
        errors = []
        if so.backstore.plugin == "ramdisk":
            path = "ramdisk"
        else:
            path = so.udev_path

        if not path:
            errors.append("BROKEN STORAGE LINK")

        legacy = []
        if self.rtsnode.name != self.name:
            legacy.append("ADDED SUFFIX")
        if len(list(self.rtsnode.backstore.storage_objects)) > 1:
            legacy.append("SHARED HBA")

        if legacy:
            errors.append("LEGACY: " + ", ".join(legacy))

        size = bytes_to_human(getattr(so, "size", 0))

        if errors:
            msg = ", ".join(errors)
            if path:
                msg += " (%s %s)" % (path, so.status)
            return (msg, False)
        else:
            return ("%s %s%s" % (path, size, so.status), True)

<commit_msg>Remove next_hba_index<commit_after>'''
Implements the targetcli backstores related UI.

This file is part of targetcli.
Copyright (c) 2011 by RisingTide Systems LLC

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, version 3 (AGPLv3).

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

from ui_node import UINode, UIRTSLibNode
from rtslib import RTSRoot
from rtslib import FileIOBackstore, BlockBackstore
from rtslib import PSCSIBackstore, RDMCPBackstore
from rtslib import FileIOStorageObject, BlockStorageObject
from rtslib import PSCSIStorageObject, RDMCPStorageObject
from rtslib.utils import (get_block_type, is_disk_partition)
from configshell import ExecutionError
import os
import re

def human_to_bytes(hsize, kilo=1024):
    '''
    This function converts human-readable amounts of bytes to bytes.
    It understands the following units :
        - I{B} or no unit present for Bytes
        - I{k}, I{K}, I{kB}, I{KB} for kB (kilobytes)
        - I{m}, I{M}, I{mB}, I{MB} for MB (megabytes)
        - I{g}, I{G}, I{gB}, I{GB} for GB (gigabytes)
        - I{t}, I{T}, I{tB}, I{TB} for TB (terabytes)

    Note: The definition of I{kilo} defaults to 1kB = 1024Bytes.
    Strictly speaking, those should not be called I{kB} but I{kiB}.
    You can override that with the optional kilo parameter.

    @param hsize: The human-readable version of the Bytes amount to convert
    @type hsize: string or int
    @param kilo: Optional base for the kilo prefix
    @type kilo: int
    @return: An int representing the human-readable string converted to bytes
    '''
    size = str(hsize).replace("g","G").replace("K","k")
    size = size.replace("m","M").replace("t","T")
    if not re.match("^[0-9]+[T|G|M|k]?[B]?$", size):
        raise RTSLibError("Cannot interpret size, wrong format: %s" % hsize)

    size = size.rstrip('B')

    units = ['k', 'M', 'G', 'T']
    try:
        power = units.index(size[-1]) + 1
    except ValueError:
        power = 0
        size = int(size)
    else:
        size = int(size[:-1])

    return size * (int(kilo) ** power)

def bytes_to_human(size, kilo=1024.0):
    if not size:
        return ""
    for x in ['bytes','K','M','G','T']:
        if size < kilo:
            return "(%3.1f%s) " % (size, x)
        size /= kilo

class UIBackstores(UINode):
    '''
    The backstores container UI.
    '''
    def __init__(self, parent):
        UINode.__init__(self, 'backstores', parent)
        self.cfs_cwd = "%s/core" % self.cfs_cwd
        self.refresh()

    def refresh(self):
        self._children = set([])
        UIPSCSIBackstore(self)
        UIRDMCPBackstore(self)
        UIFileIOBackstore(self)
        UIBlockBackstore(self)

class UIBackstore(UINode):
    '''
    A backstore UI.
    '''
    def __init__(self, plugin, parent):
        UINode.__init__(self, plugin, parent)
        self.cfs_cwd = "%s/core" % self.cfs_cwd
        self.refresh()

    def refresh(self):
        self._children = set([])
        for so in RTSRoot().storage_objects:
            if so.backstore.plugin == self.name:
                ui_so = UIStorageObject(so, self)
                ui_so.name = so.name

    def summary(self):
        no_storage_objects = len(self._children)
        if no_storage_objects > 1:
            msg = "%d Storage Objects" % no_storage_objects
        else:
            msg = "%d Storage Object" % no_storage_objects
        return (msg, None)

    def prm_buffered(self, buffered):
        buffered = \
                self.ui_eval_param(buffered, 'bool', True)
        if buffered:
            self.shell.log.info("Using buffered mode.")
        else:
            self.shell.log.info("Not using buffered mode.")
        return buffered

    def ui_command_delete(self, name):
        '''
        Recursively deletes the storage object having the specified I{name}. If
        there are LUNs using this storage object, they will be deleted too.

        EXAMPLE
        =======
        B{delete mystorage}
        -------------------
        Deletes the storage object named mystorage, and all associated LUNs.
        '''
        self.assert_root()
        try:
            child = self.get_child(name)
        except ValueError:
            self.shell.log.error("No storage object named %s." % name)
        else:
            hba = child.rtsnode.backstore
            child.rtsnode.delete()
            if not list(hba.storage_objects):
                hba.delete()
            self.remove_child(child)
            self.shell.log.info("Deleted storage object %s." % name)
            self.parent.parent.refresh()

    def ui_complete_delete(self, parameters, text, current_param):
        '''
        Parameter auto-completion method for user command delete.
        @param parameters: Parameters on the command line.
        @type parameters: dict
        @param text: Current text of parameter being typed by the user.
        @type text: str
        @param current_param: Name of parameter to complete.
        @type current_param: str
        @return: Possible completions
        @rtype: list of str
        '''
        if current_param == 'name':
            names = [child.name for child in self.children]
            completions = [name for name in names
                           if name.startswith(text)]
        else:
            completions = []

        if len(completions) == 1:
            return [completions[0] + ' ']
        else:
            return completions

    def assert_available_so_name(self, name):
        names = [child.name for child in self.children]
        if name in names:
            raise ExecutionError("Storage object %s/%s already exist."
                                 % (self.name, name))

class UIPSCSIBackstore(UIBackstore):
    '''
    PSCSI backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'pscsi', parent)

    def ui_command_create(self, name, dev):
        '''
        Creates a PSCSI storage object, with supplied name and SCSI device. The
        SCSI device I{dev} can either be a path name to the device, in which
        case it is recommended to use the /dev/disk/by-id hierarchy to have
        consistent naming should your physical SCSI system be modified, or an
        SCSI device ID in the H:C:T:L format, which is not recommended as SCSI
        IDs may vary in time.
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = PSCSIBackstore(mode='create')

        if get_block_type(dev) is not None or is_disk_partition(dev):
            self.shell.log.info("Note: block backstore recommended for "
                                "SCSI block devices")

        try:
            so = PSCSIStorageObject(backstore, name, dev)
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created pscsi storage object %s using %s"
                            % (name, dev))
        return self.new_node(ui_so)

class UIRDMCPBackstore(UIBackstore):
    '''
    RDMCP backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'ramdisk', parent)

    def ui_command_create(self, name, size):
        '''
        Creates an RDMCP storage object. I{size} is the size of the ramdisk.

        SIZE SYNTAX
        ===========
        - If size is an int, it represents a number of bytes.
        - If size is a string, the following units can be used:
            - B{B} or no unit present for bytes
            - B{k}, B{K}, B{kB}, B{KB} for kB (kilobytes)
            - B{m}, B{M}, B{mB}, B{MB} for MB (megabytes)
            - B{g}, B{G}, B{gB}, B{GB} for GB (gigabytes)
            - B{t}, B{T}, B{tB}, B{TB} for TB (terabytes)
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = RDMCPBackstore(mode='create')
        try:
            so = RDMCPStorageObject(backstore, name, human_to_bytes(size))
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created ramdisk %s with size %s."
                            % (name, size))
        return self.new_node(ui_so)

class UIFileIOBackstore(UIBackstore):
    '''
    FileIO backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'fileio', parent)

    def _create_file(self, filename, size, sparse=True):
        f = open(filename, "w+")
        try:
            if sparse:
                f.seek(size-1)
                f.write("\0")
            else:
                self.shell.log.info("Writing %s bytes" % size)
                while size > 0:
                    write_size = min(size, 1024)
                    f.write("\0" * write_size)
                    size -= write_size
        except IOError:
            f.close()
            os.remove(filename)
            raise ExecutionError("Could not expand file to size")
        f.close()

    def ui_command_create(self, name, file_or_dev, size=None, buffered=None,
                          sparse=None):
        '''
        Creates a FileIO storage object. If I{file_or_dev} is a path to a
        regular file to be used as backend, then the I{size} parameter is
        mandatory. Else, if I{file_or_dev} is a path to a block device, the
        size parameter B{must} be ommited. If present, I{size} is the size of
        the file to be used, I{file} the path to the file or I{dev} the path to
        a block device. The I{buffered} parameter is a boolean stating
        whether or not to enable buffered mode. It is enabled by default
        (asynchronous mode). The I{sparse} parameter is only applicable when
        creating a new backing file. It is a boolean stating if the
        created file should be created as a sparse file (the default), or
        fully initialized.

        SIZE SYNTAX
        ===========
        - If size is an int, it represents a number of bytes.
        - If size is a string, the following units can be used:
            - B{B} or no unit present for bytes
            - B{k}, B{K}, B{kB}, B{KB} for kB (kilobytes)
            - B{m}, B{M}, B{mB}, B{MB} for MB (megabytes)
            - B{g}, B{G}, B{gB}, B{GB} for GB (gigabytes)
            - B{t}, B{T}, B{tB}, B{TB} for TB (terabytes)
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        self.shell.log.debug("Using params size=%s buffered=%s sparse=%s"
                             % (size, buffered, sparse))

        sparse = self.ui_eval_param(sparse, 'bool', True)

        backstore = FileIOBackstore(mode='create')

        is_dev = get_block_type(file_or_dev) is not None \
                or is_disk_partition(file_or_dev)

        if is_dev:
            if size:
                self.shell.log.info("Block device, size parameter ignored")
                size = None
            self.shell.log.info("Note: block backstore preferred for best results")
        else:
            # use given file size only if backing file does not exist
            if os.path.isfile(file_or_dev):
                new_size = os.path.getsize(file_or_dev)
                if size:
                    self.shell.log.info("%s exists, using its size (%s bytes) instead"
                                        % (file_or_dev, new_size))
                size = new_size
            else:
                # create file and extend to given file size
                if not size:
                    raise ExecutionError("Attempting to create file for new" +
                                         " fileio backstore, need a size")
                size = human_to_bytes(size)
                self._create_file(file_or_dev, size, sparse)

        try:
            so = FileIOStorageObject(
                backstore, name, file_or_dev,
                size,
                buffered_mode=self.prm_buffered(buffered))
        except:
            backstore.delete()
            raise

        self.shell.log.info("Created fileio %s with size %s"
                            % (name, size))
        ui_so = UIStorageObject(so, self)

        return self.new_node(ui_so)

class UIBlockBackstore(UIBackstore):
    '''
    Block backstore UI.
    '''
    def __init__(self, parent):
        UIBackstore.__init__(self, 'block', parent)

    def ui_command_create(self, name, dev):
        '''
        Creates an Block Storage object. I{dev} is the path to the TYPE_DISK
        block device to use.
        '''
        self.assert_root()
        self.assert_available_so_name(name)
        backstore = BlockBackstore(mode='create')
        try:
            so = BlockStorageObject(backstore, name, dev)
        except:
            backstore.delete()
            raise
        ui_so = UIStorageObject(so, self)
        self.shell.log.info("Created block storage object %s using %s."
                            % (name, dev))
        return self.new_node(ui_so)

class UIStorageObject(UIRTSLibNode):
    '''
    A storage object UI.
    '''
    def __init__(self, storage_object, parent):
        name = storage_object.name
        UIRTSLibNode.__init__(self, name, storage_object, parent)
        self.cfs_cwd = storage_object.path
        self.refresh()

    def ui_command_version(self):
        '''
        Displays the version of the current backstore's plugin.
        '''
        backstore = self.rtsnode.backstore
        self.shell.con.display("Backstore plugin %s %s"
                               % (backstore.plugin, backstore.version))

    def summary(self):
        so = self.rtsnode
        errors = []
        if so.backstore.plugin == "ramdisk":
            path = "ramdisk"
        else:
            path = so.udev_path

        if not path:
            errors.append("BROKEN STORAGE LINK")

        legacy = []
        if self.rtsnode.name != self.name:
            legacy.append("ADDED SUFFIX")
        if len(list(self.rtsnode.backstore.storage_objects)) > 1:
            legacy.append("SHARED HBA")

        if legacy:
            errors.append("LEGACY: " + ", ".join(legacy))

        size = bytes_to_human(getattr(so, "size", 0))

        if errors:
            msg = ", ".join(errors)
            if path:
                msg += " (%s %s)" % (path, so.status)
            return (msg, False)
        else:
            return ("%s %s%s" % (path, size, so.status), True)

<|endoftext|>


<commit_before>from __future__ import absolute_import

import datetime
import math
import operator
import warnings

import lxml.builder
import lxml.etree

from . import dates

try:
    import pytz
except ImportError:
    warnings.warn(
        "pytz not found; cannot do timezone conversions for Solr DateFields",
        ImportWarning)
    pytz = None

E = lxml.builder.ElementMaker()

class SolrError(Exception):
    pass

class solr_date(object):
    """This class can be initialized from either native python datetime
    objects and mx.DateTime objects, and will serialize to a format
    appropriate for Solr"""
    def __init__(self, v):
        if isinstance(v, solr_date):
            self._dt_obj = v._dt_obj
        elif isinstance(v, basestring):
            self._dt_obj = dates.datetime_from_w3_datestring(v)
        elif hasattr(v, "strftime"):
            self._dt_obj = self.from_date(v)
        else:
            raise TypeError("Cannot initialize solr_date from %s object"
                            % type(v))

    @staticmethod
    def from_date(dt_obj):
        # Python datetime objects may include timezone information
        if hasattr(dt_obj, 'tzinfo') and dt_obj.tzinfo:
            # but Solr requires UTC times.
            if pytz:
                return dt_obj.astimezone(pytz.utc)
            else:
                raise EnvironmentError("pytz not available, cannot do timezone conversions")
        else:
            return dt_obj

    @property
    def microsecond(self):
        if hasattr(self._dt_obj, "microsecond"):
            return self._dt_obj.microsecond
        else:
            return int(1000000*math.modf(self._dt_obj.second)[0])

    def __repr__(self):
        return repr(self._dt_obj)

    def __unicode__(self):
        """ Serialize a datetime object in the format required
        by Solr. See http://wiki.apache.org/solr/IndexingDates
        """
        return u"%s.%sZ" % (self._dt_obj.strftime("%Y-%m-%dT%H:%M:%S"),
                            "%06d" % self.microsecond)

class SolrField(object):
    def __init__(self, node, data_type):
        self.name = node.attrib["name"]
        self.multi_valued = node.attrib.get("multiValued") == "true"
        self.required = node.attrib.get("required") == "true"
        self.type = data_type

    def normalize(self, value):
        try:
            return self.type(value)
        except (TypeError, UnicodeError):
            raise SolrError("Cannot serialize %s as type %s"
                            % (self.name, value))

    def serialize(self, value):
        if hasattr(value, "__iter__"):
            if not self.multi_valued:
                raise SolrError("'%s' is not a multi-valued field" % self.name)
            return [self.serialize(v) for v in value]
        value = self.normalize(value)
        if self.type is unicode:
            return value
        elif self.type is bool:
            return u"true" if value else u"false"
        else:
            return unicode(value)

class SolrSchema(object):
    solr_data_types = {
        'solr.StrField':unicode,
        'solr.TextField':unicode,
        'solr.BoolField':bool,
        'solr.IntField':int,
        'solr.SortableIntField':int,
        'solr.LongField':long,
        'solr.SortableLongField':long,
        'solr.FloatField':float,
        'solr.SortableFloatField':float,
        'solr.DoubleField':float,
        'solr.SortableDoubleField':float,
        'solr.DateField':solr_date
        }

    def __init__(self, f):
        """initialize a schema object from a
        filename or file-like object."""
        self.fields, self.default_field_name, self.unique_key \
            = self.schema_parse(f)
        self.default_field = self.fields[self.default_field_name] \
            if self.default_field_name else None
        self.unique_field = self.fields[self.unique_key] \
            if self.unique_key else None

    def schema_parse(self, f):
        try:
            schemadoc = lxml.etree.parse(f)
        except lxml.etree.XMLSyntaxError, e:
            raise SolrError("Invalid XML in schema:\n%s" % e.args[0])
        field_types = {}
        for data_type, t in self.solr_data_types.items():
            for field_type in schemadoc.xpath("/schema/types/fieldType[@class='%s']/@name" % data_type):
                field_types[field_type] = t
        fields = {}
        for field in schemadoc.xpath("/schema/fields/field"):
            try:
                name, type = field.attrib['name'], field.attrib['type']
            except KeyError, e:
                raise SolrError("Invalid schema.xml: missing %s attribute on field" % e.message)
            try:
                data_type = field_types[type]
            except KeyError, e:
                raise SolrError("Invalid schema.xml: %s field_type undefined" % type)
            fields[name] = SolrField(field, data_type)
        default_field_name = schemadoc.xpath("/schema/defaultSearchField")
        default_field_name = default_field_name[0].text \
            if default_field_name else None
        unique_key = schemadoc.xpath("/schema/uniqueKey")
        unique_key = unique_key[0].text if unique_key else None
        return fields, default_field_name, unique_key

    def missing_fields(self, field_names):
        return [name for name in set(self.fields.keys()) - set(field_names)
                if self.fields[name].required]

    def serialize_value(self, k, v):
        if not k in self.fields:
            raise SolrError("No such field '%s' in current schema" % k)
        return self.fields[k].serialize(v)

    def get_id_for_doc(self, doc):
        if not self.unique_key:
            raise SolrError("Schema has no unique key")
        if self.unique_key not in doc:
            raise SolrError("doc doesn't contain unique_key %s"
                            % self.unique_key)
        id = doc[self.unique_key]
        return self.unique_field.serialize(id)

    def make_update(self, docs):
        return SolrUpdate(self, docs)

    def make_delete(self, docs, query):
        return SolrDelete(self, docs, query)

    def parse_results(self, msg):
        return SolrResults(self, msg)

class SolrUpdate(object):
    ADD = E.add
    DOC = E.doc
    FIELD = E.field

    def __init__(self, schema, docs):
        self.schema = schema
        self.xml = self.add(docs)

    def fields(self, name, values):
        return [self.FIELD({'name':name}, value)
                for value in self.schema.serialize_value(name, values)]

    def doc(self, doc):
        missing_fields = self.schema.missing_fields(doc.keys())
        if missing_fields:
            raise SolrError("These required fields are unspecified:\n %s" %
                            missing_fields)
        if not doc:
            return self.DOC()
        else:
            return self.DOC(*reduce(operator.add,
                                    [self.fields(name, values)
                                     for name, values in doc.items()]))

    def add(self, docs):
        if hasattr(docs, "items") or not hasattr(docs, "__iter__"):
            docs = [docs]
        schema_fields = self.schema.fields.keys()
        docs = [(doc if hasattr(doc, "items")
                 else object_to_dict(doc, schema_fields))
                for doc in docs]
        return self.ADD(*[self.doc(doc) for doc in docs])

    def __str__(self):
        return lxml.etree.tostring(self.xml, encoding='utf-8')

class SolrDelete(object):
    DELETE = E.delete
    ID = E.id
    QUERY = E.query
    def __init__(self, schema, docs=None, queries=None):
        self.schema = schema
        deletions = self.delete_docs(docs)
        deletions += self.delete_queries(queries)
        self.xml = self.DELETE(*deletions)

    def delete_docs(self, docs):
        if not self.schema.unique_key:
            raise SolrError("This schema has no unique key - you can only delete by query")
        if docs is None:
            docs = []
        deletions = []
        if not hasattr(docs, "__iter__") or hasattr(docs, "items"):
            docs = [docs]
        for doc in docs:
            # Really this next should check the expected type of unique key
            if isinstance(doc, (basestring, int, long, float)):
                # and what about dates?
                v = self.schema.unique_field.serialize(doc)
                deletions.append(self.ID(v))
            else:
                doc = doc if hasattr(doc, "items") \
                    else object_to_dict(doc, self.schema.fields.keys())
                deletions.append(self.ID(self.schema.get_id_for_doc(doc)))
        return deletions

    def delete_queries(self, queries):
        if queries is None:
            return []
        if isinstance(queries, basestring):
            queries = [queries]
        return [self.QUERY(query) for query in queries]

    def __str__(self):
        return lxml.etree.tostring(self.xml, encoding='utf-8')

class SolrFacetCounts(object):
    members= ["facet_dates", "facet_fields", "facet_queries"]
    def __init__(self, **kwargs):
        for member in self.members:
            setattr(self, member, kwargs.get(member, ()))
        self.facet_fields = dict(self.facet_fields)

    @classmethod
    def from_response(cls, response):
        facet_counts_dict = dict(response.get("facet_counts", {}))
        return SolrFacetCounts(**facet_counts_dict)

class SolrResults(object):
    def __init__(self, schema, xmlmsg):
        self.schema = schema
        doc = lxml.etree.fromstring(xmlmsg)
        details = dict(value_from_node(n) for n in
                       doc.xpath("/response/lst[@name!='moreLikeThis']"))
        details['responseHeader'] = dict(details['responseHeader'])
        for attr in ["QTime", "params", "status"]:
            setattr(self, attr, details['responseHeader'].get(attr))
        if self.status != 0:
            raise ValueError("Response indicates an error")
        result_node = doc.xpath("/response/result")[0]
        self.result = SolrResult(result_node)
        self.facet_counts = SolrFacetCounts.from_response(details)
        self.highlighting = dict((k, dict(v))
                                 for k, v in details.get("highlighting", ()))
        more_like_these_nodes = \
            doc.xpath("/response/lst[@name='moreLikeThis']/result")
        more_like_these_results = [SolrResult(node)
                                  for node in more_like_these_nodes]
        self.more_like_these = dict((n.name, n)
                                         for n in more_like_these_results)
        if len(self.more_like_these) == 1:
            self.more_like_this = self.more_like_these.values()[0]
        else:
            self.more_like_this = None

    def __str__(self):
        return str(self.result)

    def __len__(self):
        return len(self.result.docs)

    def __getitem__(self, key):
        return self.result.docs[key]

class SolrResult(object):
    def __init__(self, node):
        self.name = node.attrib['name']
        self.numFound = node.attrib['numFound']
        self.start = node.attrib['start']
        self.docs = [value_from_node(n) for n in node.xpath("doc")]

    def __str__(self):
        return "%(numFound)s results found, starting at #%(start)s\n\n" % self.__dict__ + str(self.docs)

def object_to_dict(o, names):
    return dict((name, getattr(o, name)) for name in names
                 if (hasattr(o, name) and getattr(o, name) is not None))

def value_from_node(node):
    name = node.attrib.get('name')
    if node.tag in ('lst', 'arr'):
        value = [value_from_node(n) for n in node.getchildren()]
    if node.tag in 'doc':
        value = dict(value_from_node(n) for n in node.getchildren())
    elif node.tag == 'null':
        value = None
    elif node.tag in ('str', 'byte'):
        value = node.text
    elif node.tag in ('short', 'int'):
        value = int(node.text)
    elif node.tag == 'long':
        value = long(node.text)
    elif node.tag == 'bool':
        value = True if node.text == "true" else False
    elif node.tag in ('float', 'double'):
        value = float(node.text)
    elif node.tag == 'date':
        value = solr_date(node.text)
    if name is not None:
        return name, value
    else:
        return value
<commit_msg>Catch more general ValueError instead of just UnicodeError for field data normalization<commit_after>from __future__ import absolute_import

import datetime
import math
import operator
import warnings

import lxml.builder
import lxml.etree

from . import dates

try:
    import pytz
except ImportError:
    warnings.warn(
        "pytz not found; cannot do timezone conversions for Solr DateFields",
        ImportWarning)
    pytz = None

E = lxml.builder.ElementMaker()

class SolrError(Exception):
    pass

class solr_date(object):
    """This class can be initialized from either native python datetime
    objects and mx.DateTime objects, and will serialize to a format
    appropriate for Solr"""
    def __init__(self, v):
        if isinstance(v, solr_date):
            self._dt_obj = v._dt_obj
        elif isinstance(v, basestring):
            self._dt_obj = dates.datetime_from_w3_datestring(v)
        elif hasattr(v, "strftime"):
            self._dt_obj = self.from_date(v)
        else:
            raise TypeError("Cannot initialize solr_date from %s object"
                            % type(v))

    @staticmethod
    def from_date(dt_obj):
        # Python datetime objects may include timezone information
        if hasattr(dt_obj, 'tzinfo') and dt_obj.tzinfo:
            # but Solr requires UTC times.
            if pytz:
                return dt_obj.astimezone(pytz.utc)
            else:
                raise EnvironmentError("pytz not available, cannot do timezone conversions")
        else:
            return dt_obj

    @property
    def microsecond(self):
        if hasattr(self._dt_obj, "microsecond"):
            return self._dt_obj.microsecond
        else:
            return int(1000000*math.modf(self._dt_obj.second)[0])

    def __repr__(self):
        return repr(self._dt_obj)

    def __unicode__(self):
        """ Serialize a datetime object in the format required
        by Solr. See http://wiki.apache.org/solr/IndexingDates
        """
        return u"%s.%sZ" % (self._dt_obj.strftime("%Y-%m-%dT%H:%M:%S"),
                            "%06d" % self.microsecond)

class SolrField(object):
    def __init__(self, node, data_type):
        self.name = node.attrib["name"]
        self.multi_valued = node.attrib.get("multiValued") == "true"
        self.required = node.attrib.get("required") == "true"
        self.type = data_type

    def normalize(self, value):
        try:
            return self.type(value)
        except (TypeError, ValueError):
            raise SolrError("Cannot serialize %s as type %s"
                            % (self.name, value))

    def serialize(self, value):
        if hasattr(value, "__iter__"):
            if not self.multi_valued:
                raise SolrError("'%s' is not a multi-valued field" % self.name)
            return [self.serialize(v) for v in value]
        value = self.normalize(value)
        if self.type is unicode:
            return value
        elif self.type is bool:
            return u"true" if value else u"false"
        else:
            return unicode(value)

class SolrSchema(object):
    solr_data_types = {
        'solr.StrField':unicode,
        'solr.TextField':unicode,
        'solr.BoolField':bool,
        'solr.IntField':int,
        'solr.SortableIntField':int,
        'solr.LongField':long,
        'solr.SortableLongField':long,
        'solr.FloatField':float,
        'solr.SortableFloatField':float,
        'solr.DoubleField':float,
        'solr.SortableDoubleField':float,
        'solr.DateField':solr_date
        }

    def __init__(self, f):
        """initialize a schema object from a
        filename or file-like object."""
        self.fields, self.default_field_name, self.unique_key \
            = self.schema_parse(f)
        self.default_field = self.fields[self.default_field_name] \
            if self.default_field_name else None
        self.unique_field = self.fields[self.unique_key] \
            if self.unique_key else None

    def schema_parse(self, f):
        try:
            schemadoc = lxml.etree.parse(f)
        except lxml.etree.XMLSyntaxError, e:
            raise SolrError("Invalid XML in schema:\n%s" % e.args[0])
        field_types = {}
        for data_type, t in self.solr_data_types.items():
            for field_type in schemadoc.xpath("/schema/types/fieldType[@class='%s']/@name" % data_type):
                field_types[field_type] = t
        fields = {}
        for field in schemadoc.xpath("/schema/fields/field"):
            try:
                name, type = field.attrib['name'], field.attrib['type']
            except KeyError, e:
                raise SolrError("Invalid schema.xml: missing %s attribute on field" % e.message)
            try:
                data_type = field_types[type]
            except KeyError, e:
                raise SolrError("Invalid schema.xml: %s field_type undefined" % type)
            fields[name] = SolrField(field, data_type)
        default_field_name = schemadoc.xpath("/schema/defaultSearchField")
        default_field_name = default_field_name[0].text \
            if default_field_name else None
        unique_key = schemadoc.xpath("/schema/uniqueKey")
        unique_key = unique_key[0].text if unique_key else None
        return fields, default_field_name, unique_key

    def missing_fields(self, field_names):
        return [name for name in set(self.fields.keys()) - set(field_names)
                if self.fields[name].required]

    def serialize_value(self, k, v):
        if not k in self.fields:
            raise SolrError("No such field '%s' in current schema" % k)
        return self.fields[k].serialize(v)

    def get_id_for_doc(self, doc):
        if not self.unique_key:
            raise SolrError("Schema has no unique key")
        if self.unique_key not in doc:
            raise SolrError("doc doesn't contain unique_key %s"
                            % self.unique_key)
        id = doc[self.unique_key]
        return self.unique_field.serialize(id)

    def make_update(self, docs):
        return SolrUpdate(self, docs)

    def make_delete(self, docs, query):
        return SolrDelete(self, docs, query)

    def parse_results(self, msg):
        return SolrResults(self, msg)

class SolrUpdate(object):
    ADD = E.add
    DOC = E.doc
    FIELD = E.field

    def __init__(self, schema, docs):
        self.schema = schema
        self.xml = self.add(docs)

    def fields(self, name, values):
        return [self.FIELD({'name':name}, value)
                for value in self.schema.serialize_value(name, values)]

    def doc(self, doc):
        missing_fields = self.schema.missing_fields(doc.keys())
        if missing_fields:
            raise SolrError("These required fields are unspecified:\n %s" %
                            missing_fields)
        if not doc:
            return self.DOC()
        else:
            return self.DOC(*reduce(operator.add,
                                    [self.fields(name, values)
                                     for name, values in doc.items()]))

    def add(self, docs):
        if hasattr(docs, "items") or not hasattr(docs, "__iter__"):
            docs = [docs]
        schema_fields = self.schema.fields.keys()
        docs = [(doc if hasattr(doc, "items")
                 else object_to_dict(doc, schema_fields))
                for doc in docs]
        return self.ADD(*[self.doc(doc) for doc in docs])

    def __str__(self):
        return lxml.etree.tostring(self.xml, encoding='utf-8')

class SolrDelete(object):
    DELETE = E.delete
    ID = E.id
    QUERY = E.query
    def __init__(self, schema, docs=None, queries=None):
        self.schema = schema
        deletions = self.delete_docs(docs)
        deletions += self.delete_queries(queries)
        self.xml = self.DELETE(*deletions)

    def delete_docs(self, docs):
        if not self.schema.unique_key:
            raise SolrError("This schema has no unique key - you can only delete by query")
        if docs is None:
            docs = []
        deletions = []
        if not hasattr(docs, "__iter__") or hasattr(docs, "items"):
            docs = [docs]
        for doc in docs:
            # Really this next should check the expected type of unique key
            if isinstance(doc, (basestring, int, long, float)):
                # and what about dates?
                v = self.schema.unique_field.serialize(doc)
                deletions.append(self.ID(v))
            else:
                doc = doc if hasattr(doc, "items") \
                    else object_to_dict(doc, self.schema.fields.keys())
                deletions.append(self.ID(self.schema.get_id_for_doc(doc)))
        return deletions

    def delete_queries(self, queries):
        if queries is None:
            return []
        if isinstance(queries, basestring):
            queries = [queries]
        return [self.QUERY(query) for query in queries]

    def __str__(self):
        return lxml.etree.tostring(self.xml, encoding='utf-8')

class SolrFacetCounts(object):
    members= ["facet_dates", "facet_fields", "facet_queries"]
    def __init__(self, **kwargs):
        for member in self.members:
            setattr(self, member, kwargs.get(member, ()))
        self.facet_fields = dict(self.facet_fields)

    @classmethod
    def from_response(cls, response):
        facet_counts_dict = dict(response.get("facet_counts", {}))
        return SolrFacetCounts(**facet_counts_dict)

class SolrResults(object):
    def __init__(self, schema, xmlmsg):
        self.schema = schema
        doc = lxml.etree.fromstring(xmlmsg)
        details = dict(value_from_node(n) for n in
                       doc.xpath("/response/lst[@name!='moreLikeThis']"))
        details['responseHeader'] = dict(details['responseHeader'])
        for attr in ["QTime", "params", "status"]:
            setattr(self, attr, details['responseHeader'].get(attr))
        if self.status != 0:
            raise ValueError("Response indicates an error")
        result_node = doc.xpath("/response/result")[0]
        self.result = SolrResult(result_node)
        self.facet_counts = SolrFacetCounts.from_response(details)
        self.highlighting = dict((k, dict(v))
                                 for k, v in details.get("highlighting", ()))
        more_like_these_nodes = \
            doc.xpath("/response/lst[@name='moreLikeThis']/result")
        more_like_these_results = [SolrResult(node)
                                  for node in more_like_these_nodes]
        self.more_like_these = dict((n.name, n)
                                         for n in more_like_these_results)
        if len(self.more_like_these) == 1:
            self.more_like_this = self.more_like_these.values()[0]
        else:
            self.more_like_this = None

    def __str__(self):
        return str(self.result)

    def __len__(self):
        return len(self.result.docs)

    def __getitem__(self, key):
        return self.result.docs[key]

class SolrResult(object):
    def __init__(self, node):
        self.name = node.attrib['name']
        self.numFound = node.attrib['numFound']
        self.start = node.attrib['start']
        self.docs = [value_from_node(n) for n in node.xpath("doc")]

    def __str__(self):
        return "%(numFound)s results found, starting at #%(start)s\n\n" % self.__dict__ + str(self.docs)

def object_to_dict(o, names):
    return dict((name, getattr(o, name)) for name in names
                 if (hasattr(o, name) and getattr(o, name) is not None))

def value_from_node(node):
    name = node.attrib.get('name')
    if node.tag in ('lst', 'arr'):
        value = [value_from_node(n) for n in node.getchildren()]
    if node.tag in 'doc':
        value = dict(value_from_node(n) for n in node.getchildren())
    elif node.tag == 'null':
        value = None
    elif node.tag in ('str', 'byte'):
        value = node.text
    elif node.tag in ('short', 'int'):
        value = int(node.text)
    elif node.tag == 'long':
        value = long(node.text)
    elif node.tag == 'bool':
        value = True if node.text == "true" else False
    elif node.tag in ('float', 'double'):
        value = float(node.text)
    elif node.tag == 'date':
        value = solr_date(node.text)
    if name is not None:
        return name, value
    else:
        return value
<|endoftext|>