<commit_before>try:
    import collections.abc as collectionsabc
except ImportError:
    import collections as collectionsabc
import json
import threading

__version__ = '1.0.2'

class Node:
    def __deepcopy__(self, memodict={}):
        return self.value()

    def __init__(self, root, key_path=None):
        if not isinstance(root, File):
            root = File(root)
        self.root = root
        self.key_path = [] if key_path is None else key_path[:]

    def __str__(self):
        return str(self.value())

    def __repr__(self):
        return 'lazyjson.Node(' + repr(self.root.file_info) + ', ' + repr(self.key_path) + ')'

    def value(self):
        return self.root.value_at_key_path(self.key_path)

class Dict(Node, collectionsabc.MutableMapping):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __iter__(self):
        for key in self.value():
            yield key

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

class List(Node, collectionsabc.MutableSequence):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

    def insert(self, index, value):
        self.root.insert_value_at_key_path(self.key_path + [index], value)

class File(Dict):
    def __init__(self, file_info):
        self.file_info = file_info
        self.lock = threading.Lock()
        super().__init__(self)

    def delete_value_at_key_path(self, key_path):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = {}
        else:
            for key in key_path[:-1]:
                item = item[key]
            del item[key_path[-1]]
        self.set(json_dict)

    def insert_value_at_key_path(self, key_path, value):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item.insert(key_path[-1], value)
        self.set(json_dict)

    def set(self, value):
        if isinstance(value, Node):
            value = value.value()
        with self.lock:
            with open(self.file_info, 'w') as json_file:
                json.dump(value, json_file, sort_keys=True, indent=4, separators=(',', ': '))
                print(file=json_file) # json.dump doesn't end the file in a newline, so add it manually

    def set_value_at_key_path(self, key_path, value):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item[key_path[-1]] = value
        self.set(json_dict)

    def value_at_key_path(self, key_path):
        with open(self.file_info) as json_file:
            ret = json.load(json_file)
        for key in key_path:
            ret = ret[key]
        return ret
<commit_msg>Support for pathlib.Path objects<commit_after>try:
    import collections.abc as collectionsabc
except ImportError:
    import collections as collectionsabc
import json
import threading

__version__ = '1.0.2'

try:
    import pathlib
except ImportError:
    pass
else:
    import builtins

    def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
        if isinstance(file, pathlib.Path):
            return file.open(mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline)
        else:
            return builtins.open(file, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd, opener=opener)

class Node:
    def __deepcopy__(self, memodict={}):
        return self.value()

    def __init__(self, root, key_path=None):
        if not isinstance(root, File):
            root = File(root)
        self.root = root
        self.key_path = [] if key_path is None else key_path[:]

    def __str__(self):
        return str(self.value())

    def __repr__(self):
        return 'lazyjson.Node(' + repr(self.root.file_info) + ', ' + repr(self.key_path) + ')'

    def value(self):
        return self.root.value_at_key_path(self.key_path)

class Dict(Node, collectionsabc.MutableMapping):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __iter__(self):
        for key in self.value():
            yield key

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

class List(Node, collectionsabc.MutableSequence):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

    def insert(self, index, value):
        self.root.insert_value_at_key_path(self.key_path + [index], value)

class File(Dict):
    def __init__(self, file_info):
        self.file_info = file_info
        self.lock = threading.Lock()
        super().__init__(self)

    def delete_value_at_key_path(self, key_path):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = {}
        else:
            for key in key_path[:-1]:
                item = item[key]
            del item[key_path[-1]]
        self.set(json_dict)

    def insert_value_at_key_path(self, key_path, value):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item.insert(key_path[-1], value)
        self.set(json_dict)

    def set(self, value):
        if isinstance(value, Node):
            value = value.value()
        with self.lock:
            with open(self.file_info, 'w') as json_file:
                json.dump(value, json_file, sort_keys=True, indent=4, separators=(',', ': '))
                print(file=json_file) # json.dump doesn't end the file in a newline, so add it manually

    def set_value_at_key_path(self, key_path, value):
        with open(self.file_info) as json_file:
            json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item[key_path[-1]] = value
        self.set(json_dict)

    def value_at_key_path(self, key_path):
        with open(self.file_info) as json_file:
            ret = json.load(json_file)
        for key in key_path:
            ret = ret[key]
        return ret
<|endoftext|>


<commit_before>import os, sys, __builtin__, tempfile, operator, pkg_resources
_os = sys.modules[os.name]
_open = open
_file = file

from distutils.errors import DistutilsError
from pkg_resources import working_set

__all__ = [
    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]

def run_setup(setup_script, args):
    """Run a distutils setup script, sandboxed in its directory"""
    old_dir = os.getcwd()
    save_argv = sys.argv[:]
    save_path = sys.path[:]
    setup_dir = os.path.abspath(os.path.dirname(setup_script))
    temp_dir = os.path.join(setup_dir,'temp')
    if not os.path.isdir(temp_dir): os.makedirs(temp_dir)
    save_tmp = tempfile.tempdir
    save_modules = sys.modules.copy()
    pr_state = pkg_resources.__getstate__()
    try:
        tempfile.tempdir = temp_dir; os.chdir(setup_dir)
        try:
            sys.argv[:] = [setup_script]+list(args)
            sys.path.insert(0, setup_dir)
            # reset to include setup dir, w/clean callback list
            working_set.__init__()
            working_set.callbacks.append(lambda dist:dist.activate())
            DirectorySandbox(setup_dir).run(
                lambda: execfile(
                    "setup.py",
                    {'__file__':setup_script, '__name__':'__main__'}
                )
            )
        except SystemExit, v:
            if v.args and v.args[0]:
                raise
            # Normal exit, just return
    finally:
        pkg_resources.__setstate__(pr_state)
        sys.modules.update(save_modules)
        for key in list(sys.modules):
            if key not in save_modules: del sys.modules[key]
        os.chdir(old_dir)
        sys.path[:] = save_path
        sys.argv[:] = save_argv
        tempfile.tempdir = save_tmp

class AbstractSandbox:
    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""

    _active = False

    def __init__(self):
        self._attrs = [
            name for name in dir(_os)
                if not name.startswith('_') and hasattr(self,name)
        ]

    def _copy(self, source):
        for name in self._attrs:
            setattr(os, name, getattr(source,name))

    def run(self, func):
        """Run 'func' under os sandboxing"""
        try:
            self._copy(self)
            __builtin__.file = self._file
            __builtin__.open = self._open
            self._active = True
            return func()
        finally:
            self._active = False
            __builtin__.open = _open
            __builtin__.file = _file
            self._copy(_os)

    def _mk_dual_path_wrapper(name):
        original = getattr(_os,name)
        def wrap(self,src,dst,*args,**kw):
            if self._active:
                src,dst = self._remap_pair(name,src,dst,*args,**kw)
            return original(src,dst,*args,**kw)
        return wrap

    for name in ["rename", "link", "symlink"]:
        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)

    def _mk_single_path_wrapper(name, original=None):
        original = original or getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
            return original(path,*args,**kw)
        return wrap

    _open = _mk_single_path_wrapper('open', _open)
    _file = _mk_single_path_wrapper('file', _file)
    for name in [
        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
        "startfile", "mkfifo", "mknod", "pathconf", "access"
    ]:
        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)

    def _mk_single_with_return(name):
        original = getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
                return self._remap_output(name, original(path,*args,**kw))
            return original(path,*args,**kw)
        return wrap

    for name in ['readlink', 'tempnam']:
        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)

    def _mk_query(name):
        original = getattr(_os,name)
        def wrap(self,*args,**kw):
            retval = original(*args,**kw)
            if self._active:
                return self._remap_output(name, retval)
            return retval
        return wrap

    for name in ['getcwd', 'tmpnam']:
        if hasattr(_os,name): locals()[name] = _mk_query(name)

    def _validate_path(self,path):
        """Called to remap or validate any path, whether input or output"""
        return path

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        return self._validate_path(path)

    def _remap_output(self,operation,path):
        """Called for path outputs"""
        return self._validate_path(path)

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        return (
            self._remap_input(operation+'-from',src,*args,**kw),
            self._remap_input(operation+'-to',dst,*args,**kw)
        )

class DirectorySandbox(AbstractSandbox):
    """Restrict operations to a single subdirectory - pseudo-chroot"""

    write_ops = dict.fromkeys([
        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    ])

    def __init__(self,sandbox):
        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
        self._prefix = os.path.join(self._sandbox,'')
        AbstractSandbox.__init__(self)

    def _violation(self, operation, *args, **kw):
        raise SandboxViolation(operation, args, kw)

    def _open(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("open", path, mode, *args, **kw)
        return _open(path,mode,*args,**kw)

    def tmpnam(self): self._violation("tmpnam")

    def _ok(self,path):
        if hasattr(_os,'devnull') and path==_os.devnull: return True
        active = self._active
        try:
            self._active = False
            realpath = os.path.normcase(os.path.realpath(path))
            if realpath==self._sandbox or realpath.startswith(self._prefix):
                return True
        finally:
            self._active = active

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        if operation in self.write_ops and not self._ok(path):
            self._violation(operation, os.path.realpath(path), *args, **kw)
        return path

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        if not self._ok(src) or not self._ok(dst):
            self._violation(operation, src, dst, *args, **kw)
        return (src,dst)

    def _file(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("file", path, mode, *args, **kw)
        return _file(path,mode,*args,**kw)

    def open(self, file, flags, mode=0777):
        """Called for low-level os.open()"""
        if flags & WRITE_FLAGS and not self._ok(file):
            self._violation("os.open", file, flags, mode)
        return _os.open(file,flags,mode)

WRITE_FLAGS = reduce(
    operator.or_, [getattr(_os, a, 0) for a in
        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
)

class SandboxViolation(DistutilsError):
    """A setup script attempted to modify the filesystem outside the sandbox"""

    def __str__(self):
        return """SandboxViolation: %s%r %s

The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.

This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand.  Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args

#
<commit_msg>Fix broken devnull check; fixes issue 93 (grafted from 4dd378f0f048fbb1906dc5e71ef736fc4f089211)<commit_after>import os, sys, __builtin__, tempfile, operator, pkg_resources
_os = sys.modules[os.name]
_open = open
_file = file

from distutils.errors import DistutilsError
from pkg_resources import working_set

__all__ = [
    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]

def run_setup(setup_script, args):
    """Run a distutils setup script, sandboxed in its directory"""
    old_dir = os.getcwd()
    save_argv = sys.argv[:]
    save_path = sys.path[:]
    setup_dir = os.path.abspath(os.path.dirname(setup_script))
    temp_dir = os.path.join(setup_dir,'temp')
    if not os.path.isdir(temp_dir): os.makedirs(temp_dir)
    save_tmp = tempfile.tempdir
    save_modules = sys.modules.copy()
    pr_state = pkg_resources.__getstate__()
    try:
        tempfile.tempdir = temp_dir; os.chdir(setup_dir)
        try:
            sys.argv[:] = [setup_script]+list(args)
            sys.path.insert(0, setup_dir)
            # reset to include setup dir, w/clean callback list
            working_set.__init__()
            working_set.callbacks.append(lambda dist:dist.activate())
            DirectorySandbox(setup_dir).run(
                lambda: execfile(
                    "setup.py",
                    {'__file__':setup_script, '__name__':'__main__'}
                )
            )
        except SystemExit, v:
            if v.args and v.args[0]:
                raise
            # Normal exit, just return
    finally:
        pkg_resources.__setstate__(pr_state)
        sys.modules.update(save_modules)
        for key in list(sys.modules):
            if key not in save_modules: del sys.modules[key]
        os.chdir(old_dir)
        sys.path[:] = save_path
        sys.argv[:] = save_argv
        tempfile.tempdir = save_tmp

class AbstractSandbox:
    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""

    _active = False

    def __init__(self):
        self._attrs = [
            name for name in dir(_os)
                if not name.startswith('_') and hasattr(self,name)
        ]

    def _copy(self, source):
        for name in self._attrs:
            setattr(os, name, getattr(source,name))

    def run(self, func):
        """Run 'func' under os sandboxing"""
        try:
            self._copy(self)
            __builtin__.file = self._file
            __builtin__.open = self._open
            self._active = True
            return func()
        finally:
            self._active = False
            __builtin__.open = _open
            __builtin__.file = _file
            self._copy(_os)

    def _mk_dual_path_wrapper(name):
        original = getattr(_os,name)
        def wrap(self,src,dst,*args,**kw):
            if self._active:
                src,dst = self._remap_pair(name,src,dst,*args,**kw)
            return original(src,dst,*args,**kw)
        return wrap

    for name in ["rename", "link", "symlink"]:
        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)

    def _mk_single_path_wrapper(name, original=None):
        original = original or getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
            return original(path,*args,**kw)
        return wrap

    _open = _mk_single_path_wrapper('open', _open)
    _file = _mk_single_path_wrapper('file', _file)
    for name in [
        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
        "startfile", "mkfifo", "mknod", "pathconf", "access"
    ]:
        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)

    def _mk_single_with_return(name):
        original = getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
                return self._remap_output(name, original(path,*args,**kw))
            return original(path,*args,**kw)
        return wrap

    for name in ['readlink', 'tempnam']:
        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)

    def _mk_query(name):
        original = getattr(_os,name)
        def wrap(self,*args,**kw):
            retval = original(*args,**kw)
            if self._active:
                return self._remap_output(name, retval)
            return retval
        return wrap

    for name in ['getcwd', 'tmpnam']:
        if hasattr(_os,name): locals()[name] = _mk_query(name)

    def _validate_path(self,path):
        """Called to remap or validate any path, whether input or output"""
        return path

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        return self._validate_path(path)

    def _remap_output(self,operation,path):
        """Called for path outputs"""
        return self._validate_path(path)

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        return (
            self._remap_input(operation+'-from',src,*args,**kw),
            self._remap_input(operation+'-to',dst,*args,**kw)
        )

class DirectorySandbox(AbstractSandbox):
    """Restrict operations to a single subdirectory - pseudo-chroot"""

    write_ops = dict.fromkeys([
        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    ])

    def __init__(self,sandbox):
        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
        self._prefix = os.path.join(self._sandbox,'')
        AbstractSandbox.__init__(self)

    def _violation(self, operation, *args, **kw):
        raise SandboxViolation(operation, args, kw)

    def _open(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("open", path, mode, *args, **kw)
        return _open(path,mode,*args,**kw)

    def tmpnam(self): self._violation("tmpnam")

    def _ok(self,path):
        if hasattr(os,'devnull') and path==os.devnull: return True
        active = self._active
        try:
            self._active = False
            realpath = os.path.normcase(os.path.realpath(path))
            if realpath==self._sandbox or realpath.startswith(self._prefix):
                return True
        finally:
            self._active = active

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        if operation in self.write_ops and not self._ok(path):
            self._violation(operation, os.path.realpath(path), *args, **kw)
        return path

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        if not self._ok(src) or not self._ok(dst):
            self._violation(operation, src, dst, *args, **kw)
        return (src,dst)

    def _file(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("file", path, mode, *args, **kw)
        return _file(path,mode,*args,**kw)

    def open(self, file, flags, mode=0777):
        """Called for low-level os.open()"""
        if flags & WRITE_FLAGS and not self._ok(file):
            self._violation("os.open", file, flags, mode)
        return _os.open(file,flags,mode)

WRITE_FLAGS = reduce(
    operator.or_, [getattr(_os, a, 0) for a in
        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
)

class SandboxViolation(DistutilsError):
    """A setup script attempted to modify the filesystem outside the sandbox"""

    def __str__(self):
        return """SandboxViolation: %s%r %s

The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.

This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand.  Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args

#
<|endoftext|>


<commit_before>from flask import (Blueprint, flash, make_response, redirect, render_template,
                   request, session, url_for)
from sumy.nlp.tokenizers import Tokenizer

from wtforms.validators import URL

from ..forms import LinkForm
from ..textrank import get_parser, run_summarizer

site = Blueprint('site', __name__)

def valid_url(raw_url):
    validator = URL()
    match = validator.regex.match(raw_url)
    if not match:
        return False
    return validator.validate_hostname(match.group('host'))

@site.route('/webhook')
def webhook():
    sig = request.headers.get("x-hub-signature", '')
    if not sig:
        return {"error": "could not validate signature"}
    return redirect(url_for('site.index'), code=200)

@site.route('/summary', methods=['POST'])
def summary():
    language = 'english'
    url = request.form.get('url', '')
    max_sent = int(request.form.get('max_sent', 10))
    tokenizer = Tokenizer(language)
    parser, meta = get_parser(url, tokenizer)
    summary = run_summarizer(parser, max_sent, language)
    session_data = dict(
        summary=summary,
        url=url,
        meta=meta,
        max_sent=max_sent
    )
    session.update(session_data)
    return redirect(url_for('site.index', _anchor='summary'))

@site.route('/', methods=['GET', 'POST'])
def index():
    return render_template('index.html',
                           url=session.get('url'),
                           meta=session.get('meta'),
                           summary=session.get('summary'),
                           max_sent=session.get('max_sent'),
                           form=LinkForm())
<commit_msg>fb opengraph<commit_after>from flask import (Blueprint, flash, jsonify, make_response, redirect,
                   render_template, request, session, url_for)
from sumy.nlp.tokenizers import Tokenizer

from wtforms.validators import URL

from ..forms import LinkForm
from ..textrank import get_parser, run_summarizer

site = Blueprint('site', __name__)

def valid_url(raw_url):
    validator = URL()
    match = validator.regex.match(raw_url)
    if not match:
        return False
    return validator.validate_hostname(match.group('host'))

@site.route('/webhook')
def webhook():
    sig = request.headers.get("x-hub-signature", '')
    if not sig:
        jsonify
        return jsonify({"error": "could not validate signature"}), 200
    return redirect(url_for('site.index'), code=200)

@site.route('/summary', methods=['POST'])
def summary():
    language = 'english'
    url = request.form.get('url', '')
    max_sent = int(request.form.get('max_sent', 10))
    tokenizer = Tokenizer(language)
    parser, meta = get_parser(url, tokenizer)
    summary = run_summarizer(parser, max_sent, language)
    session_data = dict(
        summary=summary,
        url=url,
        meta=meta,
        max_sent=max_sent
    )
    session.update(session_data)
    return redirect(url_for('site.index', _anchor='summary'))

@site.route('/', methods=['GET', 'POST'])
def index():
    return render_template('index.html',
                           url=session.get('url'),
                           meta=session.get('meta'),
                           summary=session.get('summary'),
                           max_sent=session.get('max_sent'),
                           form=LinkForm())
<|endoftext|>


<commit_before>import os, sys, __builtin__, tempfile, operator
_os = sys.modules[os.name]
try:
    _file = file
except NameError:
    _file = None
_open = open
from distutils.errors import DistutilsError
__all__ = [
    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]
def run_setup(setup_script, args):
    """Run a distutils setup script, sandboxed in its directory"""
    old_dir = os.getcwd()
    save_argv = sys.argv[:]
    save_path = sys.path[:]
    setup_dir = os.path.abspath(os.path.dirname(setup_script))
    temp_dir = os.path.join(setup_dir,'temp')
    if not os.path.isdir(temp_dir): os.makedirs(temp_dir)
    save_tmp = tempfile.tempdir
    save_modules = sys.modules.copy()
    try:
        tempfile.tempdir = temp_dir
        os.chdir(setup_dir)
        try:
            sys.argv[:] = [setup_script]+list(args)
            sys.path.insert(0, setup_dir)
            DirectorySandbox(setup_dir).run(
                lambda: execfile(
                    "setup.py",
                    {'__file__':setup_script, '__name__':'__main__'}
                )
            )
        except SystemExit, v:
            if v.args and v.args[0]:
                raise
            # Normal exit, just return
    finally:
        sys.modules.update(save_modules)
        for key in list(sys.modules):
            if key not in save_modules: del sys.modules[key]
        os.chdir(old_dir)
        sys.path[:] = save_path
        sys.argv[:] = save_argv
        tempfile.tempdir = save_tmp

class AbstractSandbox:
    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""

    _active = False

    def __init__(self):
        self._attrs = [
            name for name in dir(_os)
                if not name.startswith('_') and hasattr(self,name)
        ]

    def _copy(self, source):
        for name in self._attrs:
            setattr(os, name, getattr(source,name))

    def run(self, func):
        """Run 'func' under os sandboxing"""
        try:
            self._copy(self)
            if _file:
                __builtin__.file = self._file
            __builtin__.open = self._open
            self._active = True
            return func()
        finally:
            self._active = False
            if _file:
                __builtin__.file = _file
            __builtin__.open = _open
            self._copy(_os)

    def _mk_dual_path_wrapper(name):
        original = getattr(_os,name)
        def wrap(self,src,dst,*args,**kw):
            if self._active:
                src,dst = self._remap_pair(name,src,dst,*args,**kw)
            return original(src,dst,*args,**kw)
        return wrap

    for name in ["rename", "link", "symlink"]:
        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)

    def _mk_single_path_wrapper(name, original=None):
        original = original or getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
            return original(path,*args,**kw)
        return wrap

    if _file:
        _file = _mk_single_path_wrapper('file', _file)
    _open = _mk_single_path_wrapper('open', _open)
    for name in [
        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
        "startfile", "mkfifo", "mknod", "pathconf", "access"
    ]:
        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)

    def _mk_single_with_return(name):
        original = getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
                return self._remap_output(name, original(path,*args,**kw))
            return original(path,*args,**kw)
        return wrap

    for name in ['readlink', 'tempnam']:
        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)

    def _mk_query(name):
        original = getattr(_os,name)
        def wrap(self,*args,**kw):
            retval = original(*args,**kw)
            if self._active:
                return self._remap_output(name, retval)
            return retval
        return wrap

    for name in ['getcwd', 'tmpnam']:
        if hasattr(_os,name): locals()[name] = _mk_query(name)

    def _validate_path(self,path):
        """Called to remap or validate any path, whether input or output"""
        return path

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        return self._validate_path(path)

    def _remap_output(self,operation,path):
        """Called for path outputs"""
        return self._validate_path(path)

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        return (
            self._remap_input(operation+'-from',src,*args,**kw),
            self._remap_input(operation+'-to',dst,*args,**kw)
        )

_EXCEPTIONS = [os.devnull,]

class DirectorySandbox(AbstractSandbox):
    """Restrict operations to a single subdirectory - pseudo-chroot"""

    write_ops = dict.fromkeys([
        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    ])

    def __init__(self, sandbox, exceptions=_EXCEPTIONS):
        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
        self._prefix = os.path.join(self._sandbox,'')
        self._exceptions = exceptions
        AbstractSandbox.__init__(self)

    def _violation(self, operation, *args, **kw):
        raise SandboxViolation(operation, args, kw)

    if _file:
        def _file(self, path, mode='r', *args, **kw):
            if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
                self._violation("file", path, mode, *args, **kw)
            return _file(path,mode,*args,**kw)

    def _open(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("open", path, mode, *args, **kw)
        return _open(path,mode,*args,**kw)

    def tmpnam(self):
        self._violation("tmpnam")

    def _ok(self,path):
        active = self._active
        try:
            self._active = False
            realpath = os.path.normcase(os.path.realpath(path))
            if (realpath in self._exceptions or realpath == self._sandbox
                or realpath.startswith(self._prefix)):
                return True
        finally:
            self._active = active

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        if operation in self.write_ops and not self._ok(path):
            self._violation(operation, os.path.realpath(path), *args, **kw)
        return path

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        if not self._ok(src) or not self._ok(dst):
            self._violation(operation, src, dst, *args, **kw)
        return (src,dst)

    def open(self, file, flags, mode=0777):
        """Called for low-level os.open()"""
        if flags & WRITE_FLAGS and not self._ok(file):
            self._violation("os.open", file, flags, mode)
        return _os.open(file,flags,mode)

WRITE_FLAGS = reduce(
    operator.or_,
    [getattr(_os, a, 0) for a in
        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
)

class SandboxViolation(DistutilsError):
    """A setup script attempted to modify the filesystem outside the sandbox"""

    def __str__(self):
        return """SandboxViolation: %s%r %s

The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.

This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand.  Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args

#
<commit_msg>Worked out a fix for failing sandbox errors in Windows. Fixes #118.<commit_after>import os, sys, __builtin__, tempfile, operator
_os = sys.modules[os.name]
try:
    _file = file
except NameError:
    _file = None
_open = open
from distutils.errors import DistutilsError
__all__ = [
    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]
def run_setup(setup_script, args):
    """Run a distutils setup script, sandboxed in its directory"""
    old_dir = os.getcwd()
    save_argv = sys.argv[:]
    save_path = sys.path[:]
    setup_dir = os.path.abspath(os.path.dirname(setup_script))
    temp_dir = os.path.join(setup_dir,'temp')
    if not os.path.isdir(temp_dir): os.makedirs(temp_dir)
    save_tmp = tempfile.tempdir
    save_modules = sys.modules.copy()
    try:
        tempfile.tempdir = temp_dir
        os.chdir(setup_dir)
        try:
            sys.argv[:] = [setup_script]+list(args)
            sys.path.insert(0, setup_dir)
            DirectorySandbox(setup_dir).run(
                lambda: execfile(
                    "setup.py",
                    {'__file__':setup_script, '__name__':'__main__'}
                )
            )
        except SystemExit, v:
            if v.args and v.args[0]:
                raise
            # Normal exit, just return
    finally:
        sys.modules.update(save_modules)
        for key in list(sys.modules):
            if key not in save_modules: del sys.modules[key]
        os.chdir(old_dir)
        sys.path[:] = save_path
        sys.argv[:] = save_argv
        tempfile.tempdir = save_tmp

class AbstractSandbox:
    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""

    _active = False

    def __init__(self):
        self._attrs = [
            name for name in dir(_os)
                if not name.startswith('_') and hasattr(self,name)
        ]

    def _copy(self, source):
        for name in self._attrs:
            setattr(os, name, getattr(source,name))

    def run(self, func):
        """Run 'func' under os sandboxing"""
        try:
            self._copy(self)
            if _file:
                __builtin__.file = self._file
            __builtin__.open = self._open
            self._active = True
            return func()
        finally:
            self._active = False
            if _file:
                __builtin__.file = _file
            __builtin__.open = _open
            self._copy(_os)

    def _mk_dual_path_wrapper(name):
        original = getattr(_os,name)
        def wrap(self,src,dst,*args,**kw):
            if self._active:
                src,dst = self._remap_pair(name,src,dst,*args,**kw)
            return original(src,dst,*args,**kw)
        return wrap

    for name in ["rename", "link", "symlink"]:
        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)

    def _mk_single_path_wrapper(name, original=None):
        original = original or getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
            return original(path,*args,**kw)
        return wrap

    if _file:
        _file = _mk_single_path_wrapper('file', _file)
    _open = _mk_single_path_wrapper('open', _open)
    for name in [
        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
        "startfile", "mkfifo", "mknod", "pathconf", "access"
    ]:
        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)

    def _mk_single_with_return(name):
        original = getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
                return self._remap_output(name, original(path,*args,**kw))
            return original(path,*args,**kw)
        return wrap

    for name in ['readlink', 'tempnam']:
        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)

    def _mk_query(name):
        original = getattr(_os,name)
        def wrap(self,*args,**kw):
            retval = original(*args,**kw)
            if self._active:
                return self._remap_output(name, retval)
            return retval
        return wrap

    for name in ['getcwd', 'tmpnam']:
        if hasattr(_os,name): locals()[name] = _mk_query(name)

    def _validate_path(self,path):
        """Called to remap or validate any path, whether input or output"""
        return path

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        return self._validate_path(path)

    def _remap_output(self,operation,path):
        """Called for path outputs"""
        return self._validate_path(path)

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        return (
            self._remap_input(operation+'-from',src,*args,**kw),
            self._remap_input(operation+'-to',dst,*args,**kw)
        )

_EXCEPTIONS = [os.devnull,]

try:
	gen_py = os.path.dirname(__import__('win32com.gen_py', fromlist=['__name__']).__file__)
	_EXCEPTIONS.append(gen_py)
except ImportError:
	pass

class DirectorySandbox(AbstractSandbox):
    """Restrict operations to a single subdirectory - pseudo-chroot"""

    write_ops = dict.fromkeys([
        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    ])

    def __init__(self, sandbox, exceptions=_EXCEPTIONS):
        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
        self._prefix = os.path.join(self._sandbox,'')
        self._exceptions = [os.path.normcase(os.path.realpath(path)) for path in exceptions]
        AbstractSandbox.__init__(self)

    def _violation(self, operation, *args, **kw):
        raise SandboxViolation(operation, args, kw)

    if _file:
        def _file(self, path, mode='r', *args, **kw):
            if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
                self._violation("file", path, mode, *args, **kw)
            return _file(path,mode,*args,**kw)

    def _open(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("open", path, mode, *args, **kw)
        return _open(path,mode,*args,**kw)

    def tmpnam(self):
        self._violation("tmpnam")

    def _ok(self,path):
        active = self._active
        try:
            self._active = False
            realpath = os.path.normcase(os.path.realpath(path))
            if (self._exempted(realpath) or realpath == self._sandbox
                or realpath.startswith(self._prefix)):
                return True
        finally:
            self._active = active

    def _exempted(self, filepath):
        exception_matches = map(filepath.startswith, self._exceptions)
        return any(exception_matches)

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        if operation in self.write_ops and not self._ok(path):
            self._violation(operation, os.path.realpath(path), *args, **kw)
        return path

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        if not self._ok(src) or not self._ok(dst):
            self._violation(operation, src, dst, *args, **kw)
        return (src,dst)

    def open(self, file, flags, mode=0777):
        """Called for low-level os.open()"""
        if flags & WRITE_FLAGS and not self._ok(file):
            self._violation("os.open", file, flags, mode)
        return _os.open(file,flags,mode)

WRITE_FLAGS = reduce(
    operator.or_,
    [getattr(_os, a, 0) for a in
        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
)

class SandboxViolation(DistutilsError):
    """A setup script attempted to modify the filesystem outside the sandbox"""

    def __str__(self):
        return """SandboxViolation: %s%r %s

The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.

This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand.  Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args

#
<|endoftext|>


<commit_before># __init__.py
# Entry point for anaconda storage formats subpackage.
#
# Copyright (C) 2009  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#
# Red Hat Author(s): Dave Lehman <dlehman@redhat.com>
#

import os

from ..util import notify_kernel
from ..util import get_sysfs_path_by_name
from ..util import run_program
from ..util import ObjectID
from ..storage_log import log_method_call
from ..errors import DeviceFormatError, DMError, FormatCreateError, FormatDestroyError, FormatSetupError, MDRaidError
from ..devicelibs.dm import dm_node_from_name
from ..devicelibs.mdraid import md_node_from_name
from ..i18n import N_
from ..size import Size

import logging
log = logging.getLogger("blivet")

device_formats = {}
def register_device_format(fmt_class):
    if not issubclass(fmt_class, DeviceFormat):
        raise ValueError("arg1 must be a subclass of DeviceFormat")

    device_formats[fmt_class._type] = fmt_class
    log.debug("registered device format class %s as %s", fmt_class.__name__,
                                                         fmt_class._type)

default_fstypes = ("ext4", "ext3", "ext2")
def get_default_filesystem_type():
    for fstype in default_fstypes:
        try:
            supported = get_device_format_class(fstype).supported
        except AttributeError:
            supported = None

        if supported:
            return fstype

    raise DeviceFormatError("None of %s is supported by your kernel" % ",".join(default_fstypes))

def getFormat(fmt_type, *args, **kwargs):
    """ Return an instance of the appropriate DeviceFormat class.

        :param fmt_type: The name of the formatting type
        :type fmt_type: str.
        :return: the format instance
        :rtype: :class:`DeviceFormat`
        :raises: ValueError

        .. note::

            Any additional arguments will be passed on to the constructor for
            the format class. See the various :class:`DeviceFormat` subclasses
            for an exhaustive list of the arguments that can be passed.
    """
    fmt_class = get_device_format_class(fmt_type)
    if not fmt_class:
        fmt_class = DeviceFormat
    fmt = fmt_class(*args, **kwargs)

    # this allows us to store the given type for formats we implement as
    # DeviceFormat.
    if fmt_type and fmt.type is None:
        # unknown type, but we can set the name of the format
        # this should add/set an instance attribute
        fmt._name = fmt_type

    log.debug("getFormat('%s') returning %s instance with object id %d",
       fmt_type, fmt.__class__.__name__, fmt.id)
    return fmt

def collect_device_format_classes():
    """ Pick up all device format classes from this directory.

        .. note::

            Modules must call :func:`register_device_format` to make format
            classes available to :func:`getFormat`.
    """
    mydir = os.path.dirname(__file__)
    myfile = os.path.basename(__file__)
    (myfile_name, _ext) = os.path.splitext(myfile)
    for module_file in os.listdir(mydir):
        (mod_name, ext) = os.path.splitext(module_file)
        if ext == ".py" and mod_name != myfile_name:
            try:
                globals()[mod_name] = __import__(mod_name, globals(), locals(), [], -1)
            except ImportError:
                log.error("import of device format module '%s' failed", mod_name)
                from traceback import format_exc
                log.debug("%s", format_exc())

def get_device_format_class(fmt_type):
    """ Return an appropriate format class.

        :param fmt_type: The name of the format type.
        :type fmt_type: str.
        :returns: The chosen DeviceFormat class
        :rtype: class.

        Returns None if no class is found for fmt_type.
    """
    if not device_formats:
        collect_device_format_classes()

    fmt = device_formats.get(fmt_type)
    if not fmt:
        for fmt_class in device_formats.values():
            if fmt_type and fmt_type == fmt_class._name:
                fmt = fmt_class
                break
            elif fmt_type in fmt_class._udevTypes:
                fmt = fmt_class
                break

    return fmt

class DeviceFormat(ObjectID):
    """ Generic device format.

        This represents the absence of recognized formatting. That could mean a
        device is uninitialized, has had zeros written to it, or contains some
        valid formatting that this module does not support.
    """
    _type = None
    _name = N_("Unknown")
    _udevTypes = []
    partedFlag = None
    partedSystem = None
    _formattable = False                # can be formatted
    _supported = False                  # is supported
    _linuxNative = False                # for clearpart
    _packages = []                      # required packages
    _services = []                      # required services
    _resizable = False                  # can be resized
    _maxSize = Size(0)                  # maximum size
    _minSize = Size(0)                  # minimum size
    _dump = False
    _check = False
    _hidden = False                     # hide devices with this formatting?
    _ksMountpoint = None

    def __init__(self, **kwargs):
        """
            :keyword device: The path to the device node.
            :type device: str
            :keyword uuid: the formatting's UUID.
            :type uuid: str
            :keyword exists: Whether the formatting exists. (default: False)
            :raises: ValueError

            .. note::

                The 'device' kwarg is required for existing formats. For non-
                existent formats, it is only necessary that the :attr:`device`
                attribute be set before the :meth:`create` method runs. Note
                that you can specify the device at the last moment by specifying
                it via the 'device' kwarg to the :meth:`create` method.
        """
        ObjectID.__init__(self)
        self.device = kwargs.get("device")
        self.uuid = kwargs.get("uuid")
        self.exists = kwargs.get("exists")
        self.options = kwargs.get("options")

        # don't worry about existence if this is a DeviceFormat instance
        #if self.__class__ is DeviceFormat:
        #    self.exists = True

    def __repr__(self):
        s = ("%(classname)s instance (%(id)s) object id %(object_id)d--\n"
             "  type = %(type)s  name = %(name)s  status = %(status)s\n"
             "  device = %(device)s  uuid = %(uuid)s  exists = %(exists)s\n"
             "  options = %(options)s  supported = %(supported)s"
             "  formattable = %(format)s  resizable = %(resize)s\n" %
             {"classname": self.__class__.__name__, "id": "%#x" % id(self),
              "object_id": self.id,
              "type": self.type, "name": self.name, "status": self.status,
              "device": self.device, "uuid": self.uuid, "exists": self.exists,
              "options": self.options, "supported": self.supported,
              "format": self.formattable, "resize": self.resizable})
        return s

    @property
    def _existence_str(self):
        exist = "existing"
        if not self.exists:
            exist = "non-existent"
        return exist

    @property
    def desc(self):
        return str(self.type)

    def __str__(self):
        return "%s %s" % (self._existence_str, self.desc)

    @property
    def dict(self):
        d = {"type": self.type, "name": self.name, "device": self.device,
             "uuid": self.uuid, "exists": self.exists,
             "options": self.options, "supported": self.supported,
             "resizable": self.resizable}
        return d

    @classmethod
    def labeling(cls):
        """Returns False by default since most formats are non-labeling."""
        return False

    @classmethod
    def labelFormatOK(cls, label):
        """Checks whether the format of the label is OK for whatever
           application is used by blivet to write a label for this format.
           If there is no application that blivet uses to write a label,
           then no format is acceptable, so must return False.

           :param str label: The label to be checked

           :rtype: bool
           :return: True if the format of the label is OK, otherwise False
        """
        # pylint: disable=unused-argument
        return cls.labeling()

    def _setLabel(self, label):
        """Sets the label for this format.

           :param label: the label for this format
           :type label: str or None

           None means no label specified, or in other words, accept the default
           label that the filesystem app may set. Once the device exists the
           label should not be None, as the device must then have some label
           even if just the empty label.

           "" means the empty label, i.e., no label.

           Some filesystems, even though they do not have a
           labeling application may be already labeled, so we allow to set
           the label of a filesystem even if a labeling application does not
           exist. This can happen with the install media, for example, where
           the filesystem on the CD has a label, but there is no labeling
           application for the Iso9660FS format.

           If a labeling application does exist, the label is not
           required to have the correct format for that application.
           The allowable format for the label may be more permissive than
           the format allowed by the labeling application.

           This method is not intended to be overridden.
        """
        self._label = label # pylint: disable=attribute-defined-outside-init

    def _getLabel(self):
        """The label for this filesystem.

           :return: the label for this device
           :rtype: str

           This method is not intended to be overridden.
        """
        return self._label

    def _setOptions(self, options):
        self._options = options # pylint: disable=attribute-defined-outside-init

    def _getOptions(self):
        return self._options

    options = property(_getOptions, _setOptions)

    def _setDevice(self, devspec):
        if devspec and not devspec.startswith("/"):
            raise ValueError("device must be a fully qualified path")
        self._device = devspec # pylint: disable=attribute-defined-outside-init

    def _getDevice(self):
        return self._device

    device = property(lambda f: f._getDevice(),
                      lambda f,d: f._setDevice(d),
                      doc="Full path the device this format occupies")

    @property
    def name(self):
        if self._name:
            name = self._name
        else:
            name = self.type
        return name

    @property
    def type(self):
        return self._type

    def notifyKernel(self):
        log_method_call(self, device=self.device,
                        type=self.type)
        if not self.device:
            return

        if self.device.startswith("/dev/mapper/"):
            try:
                name = dm_node_from_name(os.path.basename(self.device))
            except DMError:
                log.warning("failed to get dm node for %s", self.device)
                return
        elif self.device.startswith("/dev/md/"):
            try:
                name = md_node_from_name(os.path.basename(self.device))
            except MDRaidError:
                log.warning("failed to get md node for %s", self.device)
                return
        else:
            name = self.device

        path = get_sysfs_path_by_name(name)
        try:
            notify_kernel(path, action="change")
        except (ValueError, IOError) as e:
            log.warning("failed to notify kernel of change: %s", e)

    def create(self, **kwargs):
        """ Write the formatting to the specified block device.

            :keyword device: path to device node
            :type device: str.
            :raises: FormatCreateError
            :returns: None.

            .. :note::

                If a device node path is passed to this method it will overwrite
                any previously set value of this instance's "device" attribute.
        """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)
        # allow late specification of device path
        device = kwargs.get("device")
        if device:
            self.device = device

        if not os.path.exists(self.device):
            raise FormatCreateError("invalid device specification", self.device)

    def destroy(self, **kwargs):
        """ Remove the formatting from the associated block device.

            :raises: FormatDestroyError
            :returns: None.
        """
        # pylint: disable=unused-argument
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)
        try:
            rc = run_program(["wipefs", "-f", "-a", self.device])
        except OSError as e:
            err = str(e)
        else:
            err = ""
            if rc:
                err = str(rc)

        if err:
            msg = "error wiping old signatures from %s: %s" % (self.device, err)
            raise FormatDestroyError(msg)

        self.exists = False

    def setup(self, **kwargs):
        """ Activate the formatting.

            :keyword device: device node path
            :type device: str.
            :raises: FormatSetupError.
            :returns: None.

            .. :note::

                If a device node path is passed to this method it will overwrite
                any previously set value of this instance's "device" attribute.
        """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)

        if not self.exists:
            raise FormatSetupError("format has not been created")

        if self.status:
            return

        # allow late specification of device path
        device = kwargs.get("device")
        if device:
            self.device = device

        if not self.device or not os.path.exists(self.device):
            raise FormatSetupError("invalid device specification")

    def teardown(self):
        """ Deactivate the formatting. """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)

    @property
    def status(self):
        return (self.exists and
                self.__class__ is not DeviceFormat and
                isinstance(self.device, str) and
                self.device and
                os.path.exists(self.device))

    @property
    def formattable(self):
        """ Can we create formats of this type? """
        return self._formattable

    @property
    def supported(self):
        """ Is this format a supported type? """
        return self._supported

    @property
    def packages(self):
        """ Packages required to manage formats of this type. """
        return self._packages

    @property
    def services(self):
        """ Services required to manage formats of this type. """
        return self._services

    @property
    def resizable(self):
        """ Can formats of this type be resized? """
        return self._resizable and self.exists

    @property
    def linuxNative(self):
        """ Is this format type native to linux? """
        return self._linuxNative

    @property
    def mountable(self):
        """ Is this something we can mount? """
        return False

    @property
    def dump(self):
        """ Whether or not this format will be dumped by dump(8). """
        return self._dump

    @property
    def check(self):
        """ Whether or not this format is checked on boot. """
        return self._check

    @property
    def maxSize(self):
        """ Maximum size for this format type. """
        return self._maxSize

    @property
    def minSize(self):
        """ Minimum size for this format instance.

            :returns: the minimum size for this format instance
            :rtype: :class:`~.size.Size`

            A value of 0 indicates an unknown size.
        """
        return self._minSize

    @property
    def hidden(self):
        """ Whether devices with this formatting should be hidden in UIs. """
        return self._hidden

    @property
    def ksMountpoint(self):
        return (self._ksMountpoint or self.type or "")

    def populateKSData(self, data):
        data.format = not self.exists
        data.fstype = self.type
        data.mountpoint = self.ksMountpoint

register_device_format(DeviceFormat)

collect_device_format_classes()
<commit_msg>Make default exists value a boolean in DeviceFormat.__init__.<commit_after># __init__.py
# Entry point for anaconda storage formats subpackage.
#
# Copyright (C) 2009  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#
# Red Hat Author(s): Dave Lehman <dlehman@redhat.com>
#

import os

from ..util import notify_kernel
from ..util import get_sysfs_path_by_name
from ..util import run_program
from ..util import ObjectID
from ..storage_log import log_method_call
from ..errors import DeviceFormatError, DMError, FormatCreateError, FormatDestroyError, FormatSetupError, MDRaidError
from ..devicelibs.dm import dm_node_from_name
from ..devicelibs.mdraid import md_node_from_name
from ..i18n import N_
from ..size import Size

import logging
log = logging.getLogger("blivet")

device_formats = {}
def register_device_format(fmt_class):
    if not issubclass(fmt_class, DeviceFormat):
        raise ValueError("arg1 must be a subclass of DeviceFormat")

    device_formats[fmt_class._type] = fmt_class
    log.debug("registered device format class %s as %s", fmt_class.__name__,
                                                         fmt_class._type)

default_fstypes = ("ext4", "ext3", "ext2")
def get_default_filesystem_type():
    for fstype in default_fstypes:
        try:
            supported = get_device_format_class(fstype).supported
        except AttributeError:
            supported = None

        if supported:
            return fstype

    raise DeviceFormatError("None of %s is supported by your kernel" % ",".join(default_fstypes))

def getFormat(fmt_type, *args, **kwargs):
    """ Return an instance of the appropriate DeviceFormat class.

        :param fmt_type: The name of the formatting type
        :type fmt_type: str.
        :return: the format instance
        :rtype: :class:`DeviceFormat`
        :raises: ValueError

        .. note::

            Any additional arguments will be passed on to the constructor for
            the format class. See the various :class:`DeviceFormat` subclasses
            for an exhaustive list of the arguments that can be passed.
    """
    fmt_class = get_device_format_class(fmt_type)
    if not fmt_class:
        fmt_class = DeviceFormat
    fmt = fmt_class(*args, **kwargs)

    # this allows us to store the given type for formats we implement as
    # DeviceFormat.
    if fmt_type and fmt.type is None:
        # unknown type, but we can set the name of the format
        # this should add/set an instance attribute
        fmt._name = fmt_type

    log.debug("getFormat('%s') returning %s instance with object id %d",
       fmt_type, fmt.__class__.__name__, fmt.id)
    return fmt

def collect_device_format_classes():
    """ Pick up all device format classes from this directory.

        .. note::

            Modules must call :func:`register_device_format` to make format
            classes available to :func:`getFormat`.
    """
    mydir = os.path.dirname(__file__)
    myfile = os.path.basename(__file__)
    (myfile_name, _ext) = os.path.splitext(myfile)
    for module_file in os.listdir(mydir):
        (mod_name, ext) = os.path.splitext(module_file)
        if ext == ".py" and mod_name != myfile_name:
            try:
                globals()[mod_name] = __import__(mod_name, globals(), locals(), [], -1)
            except ImportError:
                log.error("import of device format module '%s' failed", mod_name)
                from traceback import format_exc
                log.debug("%s", format_exc())

def get_device_format_class(fmt_type):
    """ Return an appropriate format class.

        :param fmt_type: The name of the format type.
        :type fmt_type: str.
        :returns: The chosen DeviceFormat class
        :rtype: class.

        Returns None if no class is found for fmt_type.
    """
    if not device_formats:
        collect_device_format_classes()

    fmt = device_formats.get(fmt_type)
    if not fmt:
        for fmt_class in device_formats.values():
            if fmt_type and fmt_type == fmt_class._name:
                fmt = fmt_class
                break
            elif fmt_type in fmt_class._udevTypes:
                fmt = fmt_class
                break

    return fmt

class DeviceFormat(ObjectID):
    """ Generic device format.

        This represents the absence of recognized formatting. That could mean a
        device is uninitialized, has had zeros written to it, or contains some
        valid formatting that this module does not support.
    """
    _type = None
    _name = N_("Unknown")
    _udevTypes = []
    partedFlag = None
    partedSystem = None
    _formattable = False                # can be formatted
    _supported = False                  # is supported
    _linuxNative = False                # for clearpart
    _packages = []                      # required packages
    _services = []                      # required services
    _resizable = False                  # can be resized
    _maxSize = Size(0)                  # maximum size
    _minSize = Size(0)                  # minimum size
    _dump = False
    _check = False
    _hidden = False                     # hide devices with this formatting?
    _ksMountpoint = None

    def __init__(self, **kwargs):
        """
            :keyword device: The path to the device node.
            :type device: str
            :keyword uuid: the formatting's UUID.
            :type uuid: str
            :keyword exists: Whether the formatting exists. (default: False)
            :raises: ValueError

            .. note::

                The 'device' kwarg is required for existing formats. For non-
                existent formats, it is only necessary that the :attr:`device`
                attribute be set before the :meth:`create` method runs. Note
                that you can specify the device at the last moment by specifying
                it via the 'device' kwarg to the :meth:`create` method.
        """
        ObjectID.__init__(self)
        self.device = kwargs.get("device")
        self.uuid = kwargs.get("uuid")
        self.exists = kwargs.get("exists", False)
        self.options = kwargs.get("options")

        # don't worry about existence if this is a DeviceFormat instance
        #if self.__class__ is DeviceFormat:
        #    self.exists = True

    def __repr__(self):
        s = ("%(classname)s instance (%(id)s) object id %(object_id)d--\n"
             "  type = %(type)s  name = %(name)s  status = %(status)s\n"
             "  device = %(device)s  uuid = %(uuid)s  exists = %(exists)s\n"
             "  options = %(options)s  supported = %(supported)s"
             "  formattable = %(format)s  resizable = %(resize)s\n" %
             {"classname": self.__class__.__name__, "id": "%#x" % id(self),
              "object_id": self.id,
              "type": self.type, "name": self.name, "status": self.status,
              "device": self.device, "uuid": self.uuid, "exists": self.exists,
              "options": self.options, "supported": self.supported,
              "format": self.formattable, "resize": self.resizable})
        return s

    @property
    def _existence_str(self):
        exist = "existing"
        if not self.exists:
            exist = "non-existent"
        return exist

    @property
    def desc(self):
        return str(self.type)

    def __str__(self):
        return "%s %s" % (self._existence_str, self.desc)

    @property
    def dict(self):
        d = {"type": self.type, "name": self.name, "device": self.device,
             "uuid": self.uuid, "exists": self.exists,
             "options": self.options, "supported": self.supported,
             "resizable": self.resizable}
        return d

    @classmethod
    def labeling(cls):
        """Returns False by default since most formats are non-labeling."""
        return False

    @classmethod
    def labelFormatOK(cls, label):
        """Checks whether the format of the label is OK for whatever
           application is used by blivet to write a label for this format.
           If there is no application that blivet uses to write a label,
           then no format is acceptable, so must return False.

           :param str label: The label to be checked

           :rtype: bool
           :return: True if the format of the label is OK, otherwise False
        """
        # pylint: disable=unused-argument
        return cls.labeling()

    def _setLabel(self, label):
        """Sets the label for this format.

           :param label: the label for this format
           :type label: str or None

           None means no label specified, or in other words, accept the default
           label that the filesystem app may set. Once the device exists the
           label should not be None, as the device must then have some label
           even if just the empty label.

           "" means the empty label, i.e., no label.

           Some filesystems, even though they do not have a
           labeling application may be already labeled, so we allow to set
           the label of a filesystem even if a labeling application does not
           exist. This can happen with the install media, for example, where
           the filesystem on the CD has a label, but there is no labeling
           application for the Iso9660FS format.

           If a labeling application does exist, the label is not
           required to have the correct format for that application.
           The allowable format for the label may be more permissive than
           the format allowed by the labeling application.

           This method is not intended to be overridden.
        """
        self._label = label # pylint: disable=attribute-defined-outside-init

    def _getLabel(self):
        """The label for this filesystem.

           :return: the label for this device
           :rtype: str

           This method is not intended to be overridden.
        """
        return self._label

    def _setOptions(self, options):
        self._options = options # pylint: disable=attribute-defined-outside-init

    def _getOptions(self):
        return self._options

    options = property(_getOptions, _setOptions)

    def _setDevice(self, devspec):
        if devspec and not devspec.startswith("/"):
            raise ValueError("device must be a fully qualified path")
        self._device = devspec # pylint: disable=attribute-defined-outside-init

    def _getDevice(self):
        return self._device

    device = property(lambda f: f._getDevice(),
                      lambda f,d: f._setDevice(d),
                      doc="Full path the device this format occupies")

    @property
    def name(self):
        if self._name:
            name = self._name
        else:
            name = self.type
        return name

    @property
    def type(self):
        return self._type

    def notifyKernel(self):
        log_method_call(self, device=self.device,
                        type=self.type)
        if not self.device:
            return

        if self.device.startswith("/dev/mapper/"):
            try:
                name = dm_node_from_name(os.path.basename(self.device))
            except DMError:
                log.warning("failed to get dm node for %s", self.device)
                return
        elif self.device.startswith("/dev/md/"):
            try:
                name = md_node_from_name(os.path.basename(self.device))
            except MDRaidError:
                log.warning("failed to get md node for %s", self.device)
                return
        else:
            name = self.device

        path = get_sysfs_path_by_name(name)
        try:
            notify_kernel(path, action="change")
        except (ValueError, IOError) as e:
            log.warning("failed to notify kernel of change: %s", e)

    def create(self, **kwargs):
        """ Write the formatting to the specified block device.

            :keyword device: path to device node
            :type device: str.
            :raises: FormatCreateError
            :returns: None.

            .. :note::

                If a device node path is passed to this method it will overwrite
                any previously set value of this instance's "device" attribute.
        """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)
        # allow late specification of device path
        device = kwargs.get("device")
        if device:
            self.device = device

        if not os.path.exists(self.device):
            raise FormatCreateError("invalid device specification", self.device)

    def destroy(self, **kwargs):
        """ Remove the formatting from the associated block device.

            :raises: FormatDestroyError
            :returns: None.
        """
        # pylint: disable=unused-argument
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)
        try:
            rc = run_program(["wipefs", "-f", "-a", self.device])
        except OSError as e:
            err = str(e)
        else:
            err = ""
            if rc:
                err = str(rc)

        if err:
            msg = "error wiping old signatures from %s: %s" % (self.device, err)
            raise FormatDestroyError(msg)

        self.exists = False

    def setup(self, **kwargs):
        """ Activate the formatting.

            :keyword device: device node path
            :type device: str.
            :raises: FormatSetupError.
            :returns: None.

            .. :note::

                If a device node path is passed to this method it will overwrite
                any previously set value of this instance's "device" attribute.
        """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)

        if not self.exists:
            raise FormatSetupError("format has not been created")

        if self.status:
            return

        # allow late specification of device path
        device = kwargs.get("device")
        if device:
            self.device = device

        if not self.device or not os.path.exists(self.device):
            raise FormatSetupError("invalid device specification")

    def teardown(self):
        """ Deactivate the formatting. """
        log_method_call(self, device=self.device,
                        type=self.type, status=self.status)

    @property
    def status(self):
        return (self.exists and
                self.__class__ is not DeviceFormat and
                isinstance(self.device, str) and
                self.device and
                os.path.exists(self.device))

    @property
    def formattable(self):
        """ Can we create formats of this type? """
        return self._formattable

    @property
    def supported(self):
        """ Is this format a supported type? """
        return self._supported

    @property
    def packages(self):
        """ Packages required to manage formats of this type. """
        return self._packages

    @property
    def services(self):
        """ Services required to manage formats of this type. """
        return self._services

    @property
    def resizable(self):
        """ Can formats of this type be resized? """
        return self._resizable and self.exists

    @property
    def linuxNative(self):
        """ Is this format type native to linux? """
        return self._linuxNative

    @property
    def mountable(self):
        """ Is this something we can mount? """
        return False

    @property
    def dump(self):
        """ Whether or not this format will be dumped by dump(8). """
        return self._dump

    @property
    def check(self):
        """ Whether or not this format is checked on boot. """
        return self._check

    @property
    def maxSize(self):
        """ Maximum size for this format type. """
        return self._maxSize

    @property
    def minSize(self):
        """ Minimum size for this format instance.

            :returns: the minimum size for this format instance
            :rtype: :class:`~.size.Size`

            A value of 0 indicates an unknown size.
        """
        return self._minSize

    @property
    def hidden(self):
        """ Whether devices with this formatting should be hidden in UIs. """
        return self._hidden

    @property
    def ksMountpoint(self):
        return (self._ksMountpoint or self.type or "")

    def populateKSData(self, data):
        data.format = not self.exists
        data.fstype = self.type
        data.mountpoint = self.ksMountpoint

register_device_format(DeviceFormat)

collect_device_format_classes()
<|endoftext|>


<commit_before>__version__ = '19.5'
<commit_msg>Bumped to 19.4.1 in preparation for next release.<commit_after>__version__ = '19.4.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# generated by wxGlade 0.2.1 on Mon Jan 27 21:13:04 2003

from wxPython.wx import *

class doubleThresholdFLTFrame(wxFrame):
    def __init__(self, *args, **kwds):
        # begin wxGlade: doubleThresholdFLTFrame.__init__
        kwds["style"] = wxCAPTION|wxMINIMIZE_BOX|wxMAXIMIZE_BOX|wxSYSTEM_MENU|wxRESIZE_BORDER
        wxFrame.__init__(self, *args, **kwds)
        self.viewFramePanel = wxPanel(self, -1)
        self.label_1 = wxStaticText(self.viewFramePanel, -1, "Lower threshold")
        self.lowerThresholdTextId  =  wxNewId()
        self.lowerThresholdText = wxTextCtrl(self.viewFramePanel, self.lowerThresholdTextId , "", style=wxTE_PROCESS_ENTER)
        self.label_2 = wxStaticText(self.viewFramePanel, -1, "Upper threshold")
        self.upperThresholdTextId  =  wxNewId()
        self.upperThresholdText = wxTextCtrl(self.viewFramePanel, self.upperThresholdTextId , "", style=wxTE_PROCESS_ENTER)
        self.replaceInCheckBox = wxCheckBox(self.viewFramePanel, -1, "Replace IN values with:")
        self.replaceInText = wxTextCtrl(self.viewFramePanel, -1, "")
        self.replaceOutCheckBox = wxCheckBox(self.viewFramePanel, -1, "Replace OUT values with:")
        self.replaceOutText = wxTextCtrl(self.viewFramePanel, -1, "")
        self.label_3 = wxStaticText(self.viewFramePanel, -1, "Output data type")
        self.outputDataTypeChoice = wxChoice(self.viewFramePanel, -1, choices=["choice 1"])

        self.__set_properties()
        self.__do_layout()
        # end wxGlade

    def __set_properties(self):
        # begin wxGlade: doubleThresholdFLTFrame.__set_properties
        self.SetTitle("Double Threshold Filter")
        self.replaceInCheckBox.SetValue(1)
        self.replaceOutCheckBox.SetValue(1)
        self.outputDataTypeChoice.SetSelection(0)
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: doubleThresholdFLTFrame.__do_layout
        sizer_1 = wxBoxSizer(wxVERTICAL)
        sizer_5 = wxBoxSizer(wxVERTICAL)
        sizer_3 = wxBoxSizer(wxHORIZONTAL)
        grid_sizer_2 = wxFlexGridSizer(2, 2, 4, 4)
        grid_sizer_1 = wxFlexGridSizer(2, 2, 4, 4)
        grid_sizer_1.Add(self.label_1, 0, wxLEFT|wxRIGHT|wxALIGN_CENTER_VERTICAL, 2)
        grid_sizer_1.Add(self.lowerThresholdText, 0, wxALIGN_CENTER_VERTICAL, 0)
        grid_sizer_1.Add(self.label_2, 0, wxLEFT|wxRIGHT|wxALIGN_CENTER_VERTICAL, 2)
        grid_sizer_1.Add(self.upperThresholdText, 0, wxALIGN_CENTER_VERTICAL, 0)
        grid_sizer_1.AddGrowableCol(1)
        sizer_5.Add(grid_sizer_1, 1, wxALL|wxEXPAND, 7)
        grid_sizer_2.Add(self.replaceInCheckBox, 0, 0, 0)
        grid_sizer_2.Add(self.replaceInText, 0, wxEXPAND, 0)
        grid_sizer_2.Add(self.replaceOutCheckBox, 0, 0, 0)
        grid_sizer_2.Add(self.replaceOutText, 0, wxEXPAND, 0)
        grid_sizer_2.AddGrowableCol(1)
        sizer_5.Add(grid_sizer_2, 0, wxLEFT|wxRIGHT|wxBOTTOM|wxEXPAND, 7)
        sizer_3.Add(self.label_3, 0, wxRIGHT|wxALIGN_CENTER_VERTICAL, 3)
        sizer_3.Add(self.outputDataTypeChoice, 1, 0, 0)
        sizer_5.Add(sizer_3, 0, wxALL|wxEXPAND, 7)
        self.viewFramePanel.SetAutoLayout(1)
        self.viewFramePanel.SetSizer(sizer_5)
        sizer_5.Fit(self.viewFramePanel)
        sizer_5.SetSizeHints(self.viewFramePanel)
        sizer_1.Add(self.viewFramePanel, 1, wxEXPAND, 0)
        self.SetAutoLayout(1)
        self.SetSizer(sizer_1)
        sizer_1.Fit(self)
        sizer_1.SetSizeHints(self)
        self.Layout()
        # end wxGlade

# end of class doubleThresholdFLTFrame

<commit_msg>Removed "\r"s.<commit_after>#!/usr/bin/env python
# generated by wxGlade 0.2.1 on Mon Jan 27 21:13:04 2003

from wxPython.wx import *

class doubleThresholdFLTFrame(wxFrame):
    def __init__(self, *args, **kwds):
        # begin wxGlade: doubleThresholdFLTFrame.__init__
        kwds["style"] = wxCAPTION|wxMINIMIZE_BOX|wxMAXIMIZE_BOX|wxSYSTEM_MENU|wxRESIZE_BORDER
        wxFrame.__init__(self, *args, **kwds)
        self.viewFramePanel = wxPanel(self, -1)
        self.label_1 = wxStaticText(self.viewFramePanel, -1, "Lower threshold")
        self.lowerThresholdTextId  =  wxNewId()
        self.lowerThresholdText = wxTextCtrl(self.viewFramePanel, self.lowerThresholdTextId , "", style=wxTE_PROCESS_ENTER)
        self.label_2 = wxStaticText(self.viewFramePanel, -1, "Upper threshold")
        self.upperThresholdTextId  =  wxNewId()
        self.upperThresholdText = wxTextCtrl(self.viewFramePanel, self.upperThresholdTextId , "", style=wxTE_PROCESS_ENTER)
        self.replaceInCheckBox = wxCheckBox(self.viewFramePanel, -1, "Replace IN values with:")
        self.replaceInText = wxTextCtrl(self.viewFramePanel, -1, "")
        self.replaceOutCheckBox = wxCheckBox(self.viewFramePanel, -1, "Replace OUT values with:")
        self.replaceOutText = wxTextCtrl(self.viewFramePanel, -1, "")
        self.label_3 = wxStaticText(self.viewFramePanel, -1, "Output data type")
        self.outputDataTypeChoice = wxChoice(self.viewFramePanel, -1, choices=["choice 1"])

        self.__set_properties()
        self.__do_layout()
        # end wxGlade

    def __set_properties(self):
        # begin wxGlade: doubleThresholdFLTFrame.__set_properties
        self.SetTitle("Double Threshold Filter")
        self.replaceInCheckBox.SetValue(1)
        self.replaceOutCheckBox.SetValue(1)
        self.outputDataTypeChoice.SetSelection(0)
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: doubleThresholdFLTFrame.__do_layout
        sizer_1 = wxBoxSizer(wxVERTICAL)
        sizer_5 = wxBoxSizer(wxVERTICAL)
        sizer_3 = wxBoxSizer(wxHORIZONTAL)
        grid_sizer_2 = wxFlexGridSizer(2, 2, 4, 4)
        grid_sizer_1 = wxFlexGridSizer(2, 2, 4, 4)
        grid_sizer_1.Add(self.label_1, 0, wxLEFT|wxRIGHT|wxALIGN_CENTER_VERTICAL, 2)
        grid_sizer_1.Add(self.lowerThresholdText, 0, wxALIGN_CENTER_VERTICAL, 0)
        grid_sizer_1.Add(self.label_2, 0, wxLEFT|wxRIGHT|wxALIGN_CENTER_VERTICAL, 2)
        grid_sizer_1.Add(self.upperThresholdText, 0, wxALIGN_CENTER_VERTICAL, 0)
        grid_sizer_1.AddGrowableCol(1)
        sizer_5.Add(grid_sizer_1, 1, wxALL|wxEXPAND, 7)
        grid_sizer_2.Add(self.replaceInCheckBox, 0, 0, 0)
        grid_sizer_2.Add(self.replaceInText, 0, wxEXPAND, 0)
        grid_sizer_2.Add(self.replaceOutCheckBox, 0, 0, 0)
        grid_sizer_2.Add(self.replaceOutText, 0, wxEXPAND, 0)
        grid_sizer_2.AddGrowableCol(1)
        sizer_5.Add(grid_sizer_2, 0, wxLEFT|wxRIGHT|wxBOTTOM|wxEXPAND, 7)
        sizer_3.Add(self.label_3, 0, wxRIGHT|wxALIGN_CENTER_VERTICAL, 3)
        sizer_3.Add(self.outputDataTypeChoice, 1, 0, 0)
        sizer_5.Add(sizer_3, 0, wxALL|wxEXPAND, 7)
        self.viewFramePanel.SetAutoLayout(1)
        self.viewFramePanel.SetSizer(sizer_5)
        sizer_5.Fit(self.viewFramePanel)
        sizer_5.SetSizeHints(self.viewFramePanel)
        sizer_1.Add(self.viewFramePanel, 1, wxEXPAND, 0)
        self.SetAutoLayout(1)
        self.SetSizer(sizer_1)
        sizer_1.Fit(self)
        sizer_1.SetSizeHints(self)
        self.Layout()
        # end wxGlade

# end of class doubleThresholdFLTFrame

<|endoftext|>


<commit_before>import subprocess
import cv2
import math
import os.path

def make_textures(src_dir, project_dir, image_list, resolution=256):
    dst_dir = project_dir + '/Textures/'
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = src_dir + image.name
        dst = dst_dir + image.name
        if not os.path.exists(dst):
            subprocess.run(['convert', '-resize', '%dx%d!' % (resolution, resolution), src, dst])

def make_textures_opencv(src_dir, project_dir, image_list, resolution=256):
    dst_dir = project_dir + '/Textures/'
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = src_dir + "/" + image.name
        dst = dst_dir + image.name
        if not os.path.exists(dst):
            src = cv2.imread(src, flags=cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH|cv2.IMREAD_IGNORE_ORIENTATION)
            height, width = src.shape[:2]
            # downscale image first
            method = cv2.INTER_AREA  # cv2.INTER_AREA
            scale = cv2.resize(src, (0,0),
                               fx=resolution/float(width),
                               fy=resolution/float(height),
                               interpolation=method)
            # convert to hsv color space
            hsv = cv2.cvtColor(scale, cv2.COLOR_BGR2HSV)
            hue,sat,val = cv2.split(hsv)
            # adaptive histogram equalization on 'value' channel
            clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
            aeq = clahe.apply(val)
            # recombine
            hsv = cv2.merge((hue,sat,aeq))
            # convert back to rgb
            result = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
            cv2.imwrite(dst, result)
            print("Texture %dx%d %s" % (resolution, resolution, dst))

def generate(image_list, ref_image=False, src_dir=".", project_dir=".", base_name="quick", version=1.0, trans=0.0, resolution=512 ):
    # make the textures if needed
    make_textures_opencv(src_dir, project_dir, image_list, resolution)

    max_roll = 30.0
    max_pitch = 30.0
    min_agl = 50.0
    min_time = 0.0 # the further into the flight hopefully the better the filter convergence

    ref_lon = None
    ref_lat = None

    # count matching images (starting with 1 to include the reference image)
    match_count = 0
    if ref_image:
        match_count += 1
    match_count += len(image_list)

    # write AC3D header
    name = project_dir
    name += "/"
    name += base_name
    if version:
        name += ("-%02d" % version)
    name += ".ac"
    f = open( name, "w" )
    f.write("AC3Db\n")
    f.write("MATERIAL \"\" rgb 1 1 1  amb 0.6 0.6 0.6  emis 0 0 0  spec 0.5 0.5 0.5  shi 10  trans %.2f\n" % (trans))
    f.write("OBJECT world\n")
    # wrong -- not a real rotation
    #   f.write("rot 1.0 0.0 0.0  0.0 0.0 1.0 0.0 1.0 0.0\n")
    f.write("kids " + str(match_count) + "\n")

    for image in image_list:
        if len(image.grid_list) == 0:
            continue

        # compute a priority function (higher priority tiles are raised up)
        #priority = (1.0-image.weight) - agl/400.0

        f.write("OBJECT poly\n")
        f.write("name \"rect\"\n")
        f.write("texture \"./Textures/" + image.name + "\"\n")
        f.write("loc 0 0 0\n")

        f.write("numvert %d\n" % len(image.grid_list))
        # output the ac3d polygon grid (note the grid list is in
        # this specific order because that is how we generated it
        # earlier
        pos = 0
        for v in image.grid_list:
            f.write( "%.3f %.3f %.3f\n" % (v[0], v[1], v[2]) )

        steps = int(math.sqrt(len(image.grid_list))) - 1
        f.write("numsurf %d\n" % steps**2)
        dx = 1.0 / float(steps)
        dy = 1.0 / float(steps)
        y = 1.0
        for j in range(steps):
            x = 0.0
            for i in range(steps):
                c = (j * (steps+1)) + i
                d = ((j+1) * (steps+1)) + i
                f.write("SURF 0x20\n")
                f.write("mat 0\n")
                f.write("refs 4\n")
                f.write("%d %.3f %.3f\n" % (d, x, y-dy))
                f.write("%d %.3f %.3f\n" % (d+1, x+dx, y-dy))
                f.write("%d %.3f %.3f\n" % (c+1, x+dx, y))
                f.write("%d %.3f %.3f\n" % (c, x, y))
                x += dx
            y -= dy
        f.write("kids 0\n")

    if ref_image:
        # reference poly
        f.write("OBJECT poly\n")
        f.write("name \"rect\"\n")
        f.write("texture \"Reference/3drc.png\"\n")
        f.write("loc 0 0 0\n")
        f.write("numvert 4\n")

        f.write(str(gul[0]) + " " + str(gul[1]) + " " + str(gul[2]-15) + "\n")
        f.write(str(gur[0]) + " " + str(gur[1]) + " " + str(gur[2]-15) + "\n")
        f.write(str(glr[0]) + " " + str(glr[1]) + " " + str(glr[2]-15) + "\n")
        f.write(str(gll[0]) + " " + str(gll[1]) + " " + str(gll[2]-15) + "\n")
        f.write("numsurf 1\n")
        f.write("SURF 0x20\n")
        f.write("mat 0\n")
        f.write("refs 4\n")
        f.write("3 0 0\n")
        f.write("2 1 0\n")
        f.write("1 1 1\n")
        f.write("0 0 1\n")
        f.write("kids 0\n")

    f.close()

<commit_msg>Only include listed group member images in AC3D model.<commit_after>import subprocess
import cv2
import math
import os.path

def make_textures(src_dir, project_dir, image_list, resolution=256):
    dst_dir = project_dir + '/Textures/'
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = src_dir + image.name
        dst = dst_dir + image.name
        if not os.path.exists(dst):
            subprocess.run(['convert', '-resize', '%dx%d!' % (resolution, resolution), src, dst])

def make_textures_opencv(src_dir, project_dir, image_list, resolution=256):
    dst_dir = project_dir + '/Textures/'
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = src_dir + "/" + image.name
        dst = dst_dir + image.name
        if not os.path.exists(dst):
            src = cv2.imread(src, flags=cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH|cv2.IMREAD_IGNORE_ORIENTATION)
            height, width = src.shape[:2]
            # downscale image first
            method = cv2.INTER_AREA  # cv2.INTER_AREA
            scale = cv2.resize(src, (0,0),
                               fx=resolution/float(width),
                               fy=resolution/float(height),
                               interpolation=method)
            # convert to hsv color space
            hsv = cv2.cvtColor(scale, cv2.COLOR_BGR2HSV)
            hue,sat,val = cv2.split(hsv)
            # adaptive histogram equalization on 'value' channel
            clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
            aeq = clahe.apply(val)
            # recombine
            hsv = cv2.merge((hue,sat,aeq))
            # convert back to rgb
            result = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
            cv2.imwrite(dst, result)
            print("Texture %dx%d %s" % (resolution, resolution, dst))

def generate(image_list, group, ref_image=False, src_dir=".", project_dir=".", base_name="quick", version=1.0, trans=0.0, resolution=512 ):
    # make the textures if needed
    make_textures_opencv(src_dir, project_dir, image_list, resolution)

    max_roll = 30.0
    max_pitch = 30.0
    min_agl = 50.0
    min_time = 0.0 # the further into the flight hopefully the better the filter convergence

    ref_lon = None
    ref_lat = None

    # count matching images (starting with 1 to include the reference image)
    match_count = 0
    if ref_image:
        match_count += 1
    match_count += len(group)

    # write AC3D header
    name = project_dir
    name += "/"
    name += base_name
    if version:
        name += ("-%02d" % version)
    name += ".ac"
    f = open( name, "w" )
    f.write("AC3Db\n")
    f.write("MATERIAL \"\" rgb 1 1 1  amb 0.6 0.6 0.6  emis 0 0 0  spec 0.5 0.5 0.5  shi 10  trans %.2f\n" % (trans))
    f.write("OBJECT world\n")
    # wrong -- not a real rotation
    #   f.write("rot 1.0 0.0 0.0  0.0 0.0 1.0 0.0 1.0 0.0\n")
    f.write("kids " + str(match_count) + "\n")

    for g in reversed(group):
        image = image_list[g]
        if len(image.grid_list) == 0:
            continue

        # compute a priority function (higher priority tiles are raised up)
        #priority = (1.0-image.weight) - agl/400.0

        f.write("OBJECT poly\n")
        f.write("name \"rect\"\n")
        f.write("texture \"./Textures/" + image.name + "\"\n")
        f.write("loc 0 0 0\n")

        f.write("numvert %d\n" % len(image.grid_list))
        # output the ac3d polygon grid (note the grid list is in
        # this specific order because that is how we generated it
        # earlier
        pos = 0
        for v in image.grid_list:
            f.write( "%.3f %.3f %.3f\n" % (v[0], v[1], v[2]) )

        steps = int(math.sqrt(len(image.grid_list))) - 1
        f.write("numsurf %d\n" % steps**2)
        dx = 1.0 / float(steps)
        dy = 1.0 / float(steps)
        y = 1.0
        for j in range(steps):
            x = 0.0
            for i in range(steps):
                c = (j * (steps+1)) + i
                d = ((j+1) * (steps+1)) + i
                f.write("SURF 0x20\n")
                f.write("mat 0\n")
                f.write("refs 4\n")
                f.write("%d %.3f %.3f\n" % (d, x, y-dy))
                f.write("%d %.3f %.3f\n" % (d+1, x+dx, y-dy))
                f.write("%d %.3f %.3f\n" % (c+1, x+dx, y))
                f.write("%d %.3f %.3f\n" % (c, x, y))
                x += dx
            y -= dy
        f.write("kids 0\n")

    if ref_image:
        # reference poly
        f.write("OBJECT poly\n")
        f.write("name \"rect\"\n")
        f.write("texture \"Reference/3drc.png\"\n")
        f.write("loc 0 0 0\n")
        f.write("numvert 4\n")

        f.write(str(gul[0]) + " " + str(gul[1]) + " " + str(gul[2]-15) + "\n")
        f.write(str(gur[0]) + " " + str(gur[1]) + " " + str(gur[2]-15) + "\n")
        f.write(str(glr[0]) + " " + str(glr[1]) + " " + str(glr[2]-15) + "\n")
        f.write(str(gll[0]) + " " + str(gll[1]) + " " + str(gll[2]-15) + "\n")
        f.write("numsurf 1\n")
        f.write("SURF 0x20\n")
        f.write("mat 0\n")
        f.write("refs 4\n")
        f.write("3 0 0\n")
        f.write("2 1 0\n")
        f.write("1 1 1\n")
        f.write("0 0 1\n")
        f.write("kids 0\n")

    f.close()

<|endoftext|>