<commit_before>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Command-line option handling
"""

import inspect
import optparse
import StringIO

from conary.lib import log, util
import sys

(NO_PARAM,   # arg may occur, no parameter
 ONE_PARAM,  # arg may occur once, req'd parameter
 OPT_PARAM,  # arg may occur once, optional parameter
 MULT_PARAM, # arg may occur N times, w/ parameter
 COUNT_PARAM, # arg may occur N times, value is the count
 ) = range(0,5)

class OptionError(Exception):
    val = 1
    def __init__(self, msg, parser):
        Exception.__init__(self, msg)
        self.parser = parser

class OptionParser(optparse.OptionParser):
    forbiddenOpts = set(str(x) for x in range(0,9))
    matchPartialOptions = False

    def __init__(self, *args, **kw):
        self.hobbleShortOpts = kw.pop('hobbleShortOpts', False)
        optparse.OptionParser.__init__(self, *args, **kw)

    def _match_long_opt(self, opt):
        match = optparse._match_abbrev(opt, self._long_opt)
        if not self.matchPartialOptions and opt != match:
            raise optparse.BadOptionError("no such option: %s" % opt)
        return match

    def error(self, msg):
        raise OptionError(msg, self)

    def _process_short_opts(self, rargs, values):
        if (self.hobbleShortOpts and
            (len(self.rargs[0]) > 2 or self.rargs[0][1] in self.forbiddenOpts)):
            self.largs.append(self.rargs.pop(0))
        else:
            return optparse.OptionParser._process_short_opts(self, rargs,
                                                             values)

    def _process_long_opt(self, rargs, values):
        if '=' in rargs[0]:
            had_explicit_value = True
        else:
            had_explicit_value = False
        opt = self._match_long_opt(rargs[0].split('=')[0])
        option = self._long_opt[opt]
        option.had_explicit_value = had_explicit_value
        return optparse.OptionParser._process_long_opt(self, rargs, values)

def optParamCallback(option, opt_str, value, parser, *args, **kw):
    value = True
    if option.had_explicit_value:
        newValue = parser.rargs[0]
        del parser.rargs[0]
        if newValue: # handle --opt= - treat like --opt
            value = True
    setattr(parser.values, option.dest, value)

def addOptions(parser, argDef, skip=None):
    for name, data in sorted(argDef.iteritems()):
        if name == skip:
            continue
        if isinstance(data, dict):
            group = optparse.OptionGroup(parser, name)
            addOptions(group, data)
            parser.add_option_group(group)
            continue
        help = ''
        shortOpt = None
        meta = None
        if isinstance(data, (list, tuple)):
            if len(data) == 3:
                shortOpt = data[0]
                data = data[1:]
            if len(data) >= 2:
                help = data[1]
                if isinstance(help, (list, tuple)):
                    help, meta = help
            paramType = data[0]
        else:
            paramType = data
        flagNames = ['--' + name]
        if shortOpt:
            flagNames.append(shortOpt)

        if paramType == NO_PARAM:
            parser.add_option(action='store_true', dest=name, help=help,
                              metavar=meta, *flagNames)
        elif paramType == ONE_PARAM:
            parser.add_option(dest=name, help=help, metavar=meta, *flagNames)
        elif paramType == OPT_PARAM:
            parser.add_option(action='callback',
                               callback=optParamCallback, dest=name,
                               type='string', nargs=0, help=help,
                               metavar=meta, *flagNames)
        elif paramType == MULT_PARAM:
            parser.add_option(action='append', dest=name, help=help,
                              metavar=meta, *flagNames)
        elif paramType == COUNT_PARAM:
            parser.add_option(action='count',
                              dest=name, help=help, metavar=meta,
                              *flagNames)

def processArgs(argDef, cfgMap, cfg, usage, argv=sys.argv):
    """Mostly backwards-compatible (with earlier conary processArgs)
       function that uses optparse as its backend.
    """
    return _processArgs(argDef, cfgMap, cfg, usage, argv)[:2]

def _processArgs(params, cfgMap, cfg, usage, argv=sys.argv, version=None,
                commonParams=None, useHelp=False, defaultGroup=None,
                interspersedArgs=True, hobbleShortOpts=False):
    argSet = {}
    # don't mangle the command line
    argv = argv[:]

    # historically, usage was generally a function to print out the usage
    # message.  We want it to be a string.  For now, we
    # convert here to allow backwards compatibility.
    if hasattr(usage, '__call__'):
        stdout = StringIO.StringIO()
        stderr = StringIO.StringIO()
        oldStdOut = sys.stdout
        oldStdErr = sys.stderr
        sys.stdout = stdout
        sys.stderr = stderr
        try:
            usage()
        except SystemExit:
            # some of these old usage functions even exit after
            # printing the usage message!
            pass
        sys.stdout = oldStdOut
        sys.stderr = oldStdErr
        usage = stdout.getvalue() + stderr.getvalue()

    if defaultGroup:
        d = params[defaultGroup]
    else:
        d = params
    d['debug'] = OPT_PARAM, 'Print helpful debugging output (use --debug=all for internal debug info)'
    d['debugger'] = (NO_PARAM, optparse.SUPPRESS_HELP)

    for (arg, name) in cfgMap.items():
        d[arg] = ONE_PARAM

    parser = getOptionParser(params, cfgMap, cfg, usage, version, useHelp,
                             defaultGroup, interspersedArgs,
                             hobbleShortOpts=hobbleShortOpts)
    argSet, otherArgs, options = getArgSet(params, parser, argv)

    configFileList = argSet.pop('config-file', [])
    if not isinstance(configFileList, list):
        configFileList = [configFileList]
    for path in configFileList:
        try:
            cfg.read(path, exception = True)
        except IOError, msg:
            raise OptionError(msg, parser)

    for (arg, name) in cfgMap.items():
	if argSet.has_key(arg):
	    cfg.configLine("%s %s" % (name, argSet[arg]))
	    del argSet[arg]

    if argSet.has_key('debugger'):
	del argSet['debugger']
	from conary.lib import debugger
	debugger.set_trace()
        sys.excepthook = util.genExcepthook(cfg.dumpStackOnError,
                                            debugCtrlC=True)

    if 'debug' in argSet:
        if argSet['debug'] is True:
            log.setVerbosity(log.DEBUG)
        else:
            log.setVerbosity(log.LOWLEVEL)
	del argSet['debug']
    else:
	log.setVerbosity(log.WARNING)

    return argSet, otherArgs, parser, options

def getOptionParser(params, cfgMap, cfg, usage, version=None, useHelp=False,
                    defaultGroup=None, interspersedArgs=True,
                    hobbleShortOpts=False):
    parser = OptionParser(usage=usage, add_help_option=useHelp, version=version,
                          hobbleShortOpts=hobbleShortOpts)
    if not interspersedArgs:
        parser.disable_interspersed_args()

    if defaultGroup in params:
        group = optparse.OptionGroup(parser, defaultGroup)
        addOptions(group, params[defaultGroup])
        parser.add_option_group(group)

    found = None
    for name, data in params.iteritems():
        if name == defaultGroup:
            continue
        if isinstance(data, dict):
            found = True
            break
        else:
            found = False

    if found is False:
        group = optparse.OptionGroup(parser, 'Command Options')
        addOptions(group, params, skip=defaultGroup)
        parser.add_option_group(group)
    else:
        addOptions(parser, params, skip=defaultGroup)

    return parser

def getArgSet(params, parser, argv=sys.argv):
    (options, otherArgs) = parser.parse_args(argv)

    argSet = {}

    for name, data in params.iteritems():
        if isinstance(data, dict):
            for name in data:
                val = getattr(options, name)
                if val is None:
                    continue
                argSet[name] = val
        else:
            val = getattr(options, name)
            if val is None:
                continue
            argSet[name] = val
    return argSet, otherArgs, options

class AbstractCommand(object):
    """
        Abstract command object to be subclassed used to represent commands
        in a command line interface.  To be used with MainHandler below.
        Assumes use of a lib.cfg.ConfigFile type configuration object.
    """
    commands = []
    paramHelp = '' # for each command will display <command> + paramHelp
                   # as part of usage.
    defaultGroup = 'Common Options' # The heading for options that aren't
                                    # put in any other group.

    docs = {} # add docs in the form 'long-option' : 'description'
              # or 'long-option' : ('description', 'KEYWORD').

    def __init__(self):
        self.parser = None

    def usage(self, errNo=1):
        if self.parser:
            self.parser.print_help()
        return errNo

    def setParser(self, parser):
        self.parser = parser

    def addParameters(self, argDef):
        pass

    def addConfigOptions(self, cfgMap, argDef):
        for name, (cfgName, paramType)  in cfgMap.items():
            # if it's a NO_PARAM
            if paramType == NO_PARAM:
                negName = 'no-' + name
                argDef[self.defaultGroup][negName] = NO_PARAM, optparse.SUPPRESS_HELP
                cfgMap[negName] = (cfgName, paramType)

            argDef[self.defaultGroup][name] = paramType

    def addDocs(self, argDef):
        """ Parse a docs dict assigned at the class level
            and add those docs to the parameters being sent to
            parseOptions.
        """
        d = {}
        for class_ in reversed(inspect.getmro(self.__class__)):
            if not hasattr(class_, 'docs'):
                continue
            d.update(class_.docs)

        commandDicts = [argDef]
        while commandDicts:
            commandDict = commandDicts.pop()
            for name, value in commandDict.items():
                if isinstance(value, dict):
                    commandDicts.append(value)
                    continue
                if name in d:
                    if not isinstance(value, (list, tuple)):
                        value = [ value ]
                    else:
                        value = list(value)
                    value.append(d[name])
                    commandDict[name] = value

    def prepare(self):
        params = {}
        cfgMap = {}
        self.addParameters(params)
        self.addConfigOptions(cfgMap, params)
        self.addDocs(params)
        return params, cfgMap

    def processConfigOptions(self, cfg, cfgMap, argSet):
        """
            Manage any config maps we've set up, converting
            assigning them to the config object.
        """
        configFileList = argSet.pop('config-file', [])
        if not isinstance(configFileList, list):
            configFileList = list(configFileList)

        for line in configFileList:
            cfg.read(path, exception=True)

        for (arg, (name, paramType)) in cfgMap.items():
            value = argSet.pop(arg, None)
            if value is not None:
                if arg.startswith('no-'):
                    value = not value

                cfg.configLine("%s %s" % (name, value))

        for line in argSet.pop('config', []):
            cfg.configLine(line)

    def runCommand(self, *args, **kw):
        raise NotImplementedError

class MainHandler(object):
    """
        Class to handle parsing and executing commands set up to use
        AbstractCommands
    """

    abstractCommand = None   # class to grab generic options from.  These
                             # can be used in front of
    commandList = []         # list of commands to support.
    name = None              # name to use when showing usage messages.
    version = '<no version>' # version to return to --version

    hobbleShortOpts = False # whether or not to allow -mn to be used, or to
                            # require -m -n.
    configClass = None

    def __init__(self):
        self._supportedCommands = {}
        for command in self.commandList:
            self._registerCommand(command)

    def _registerCommand(self, commandClass):
        supportedCommands = self._supportedCommands
        inst = commandClass()
        if isinstance(commandClass.commands, str):
            supportedCommands[commandClass.commands] = inst
        else:
            for cmdName in commandClass.commands:
                supportedCommands[cmdName] = inst

    def _getPreCommandOptions(self, argv, cfg):
        """Allow the user to specify generic flags before they specify the
           command to run.
        """
        thisCommand = self.abstractCommand()
        params, cfgMap = thisCommand.prepare()
        defaultGroup = thisCommand.defaultGroup
        argSet, otherArgs, parser, optionSet = _processArgs(
                                                    params, {}, cfg,
                                                    self.usage,
                                                    argv=argv[1:],
                                                    version=self.version,
                                                    useHelp=True,
                                                    defaultGroup=defaultGroup,
                                                    interspersedArgs=False,
                                    hobbleShortOpts=self.hobbleShortOpts)
        return argSet, [argv[0]] + otherArgs

    def getConfigFile(self, argv):
        """
            Find the appropriate config file
        """
        if not self.configClass:
            raise RuntimeError, ('Must define a configClass to use with this'
                                 ' main handler')
        if '--skip-default-config' in argv:
            argv.remove('--skip-default-config')
            ccfg = self.configClass(readConfigFiles=False)
        else:
            ccfg = self.configClass(readConfigFiles=True)
        return ccfg

    def main(self, argv=sys.argv, debuggerException=Exception,
             cfg=None, **kw):
        """
            Process argv and execute commands as specified.
        """

        from conary import versions
        supportedCommands = self._supportedCommands

        if cfg is None:
            cfg = self.getConfigFile(argv)

        if '--version' in argv or '-v' in argv:
            print self.version
            return

        try:
            argSet, argv = self._getPreCommandOptions(argv, cfg)
        except debuggerException:
            raise
        except OptionError, e:
            self.usage()
            print >>sys.stderr, e
            sys.exit(e.val)

        if len(argv) < 2:
            # no command specified
            return self.usage()

        commandName = argv[1]
        if commandName not in self._supportedCommands:
            return self.usage()

        thisCommand = self._supportedCommands[commandName]
        params, cfgMap = thisCommand.prepare()
        defaultGroup = thisCommand.defaultGroup
        if self.name:
            progName = self.name
        else:
            progName = argv[0]
        commandUsage = '%s %s %s' % (progName, commandName,
                                     thisCommand.paramHelp)
        try:
            newArgSet, otherArgs, parser, optionSet = _processArgs(
                                        params, {}, cfg,
                                        commandUsage,
                                        argv=argv,
                                        version=self.version,
                                        useHelp=True,
                                        defaultGroup=defaultGroup,
                                        hobbleShortOpts=self.hobbleShortOpts)
        except debuggerException, e:
            raise
        except OptionError, e:
            e.parser.print_help()
            print >> sys.stderr, e
            sys.exit(e.val)
        except versions.ParseError, e:
            print >> sys.stderr, e
            sys.exit(1)

        thisCommand.setParser(parser)
        argSet.update(newArgSet)
        thisCommand.processConfigOptions(cfg, cfgMap, argSet)
        self.runCommand(thisCommand, cfg, argSet, otherArgs, **kw)

    def runCommand(self, thisCommand, *args, **kw):
        thisCommand.runCommand(*args, **kw)

<commit_msg>whoops, fix direction of test<commit_after>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Command-line option handling
"""

import inspect
import optparse
import StringIO

from conary.lib import log, util
import sys

(NO_PARAM,   # arg may occur, no parameter
 ONE_PARAM,  # arg may occur once, req'd parameter
 OPT_PARAM,  # arg may occur once, optional parameter
 MULT_PARAM, # arg may occur N times, w/ parameter
 COUNT_PARAM, # arg may occur N times, value is the count
 ) = range(0,5)

class OptionError(Exception):
    val = 1
    def __init__(self, msg, parser):
        Exception.__init__(self, msg)
        self.parser = parser

class OptionParser(optparse.OptionParser):
    forbiddenOpts = set(str(x) for x in range(0,9))
    matchPartialOptions = False

    def __init__(self, *args, **kw):
        self.hobbleShortOpts = kw.pop('hobbleShortOpts', False)
        optparse.OptionParser.__init__(self, *args, **kw)

    def _match_long_opt(self, opt):
        match = optparse._match_abbrev(opt, self._long_opt)
        if not self.matchPartialOptions and opt != match:
            raise optparse.BadOptionError("no such option: %s" % opt)
        return match

    def error(self, msg):
        raise OptionError(msg, self)

    def _process_short_opts(self, rargs, values):
        if (self.hobbleShortOpts and
            (len(self.rargs[0]) > 2 or self.rargs[0][1] in self.forbiddenOpts)):
            self.largs.append(self.rargs.pop(0))
        else:
            return optparse.OptionParser._process_short_opts(self, rargs,
                                                             values)

    def _process_long_opt(self, rargs, values):
        if '=' in rargs[0]:
            had_explicit_value = True
        else:
            had_explicit_value = False
        opt = self._match_long_opt(rargs[0].split('=')[0])
        option = self._long_opt[opt]
        option.had_explicit_value = had_explicit_value
        return optparse.OptionParser._process_long_opt(self, rargs, values)

def optParamCallback(option, opt_str, value, parser, *args, **kw):
    value = True
    if option.had_explicit_value:
        newValue = parser.rargs[0]
        del parser.rargs[0]
        if newValue: # handle --opt= - treat like --opt
            value = newValue
    setattr(parser.values, option.dest, value)

def addOptions(parser, argDef, skip=None):
    for name, data in sorted(argDef.iteritems()):
        if name == skip:
            continue
        if isinstance(data, dict):
            group = optparse.OptionGroup(parser, name)
            addOptions(group, data)
            parser.add_option_group(group)
            continue
        help = ''
        shortOpt = None
        meta = None
        if isinstance(data, (list, tuple)):
            if len(data) == 3:
                shortOpt = data[0]
                data = data[1:]
            if len(data) >= 2:
                help = data[1]
                if isinstance(help, (list, tuple)):
                    help, meta = help
            paramType = data[0]
        else:
            paramType = data
        flagNames = ['--' + name]
        if shortOpt:
            flagNames.append(shortOpt)

        if paramType == NO_PARAM:
            parser.add_option(action='store_true', dest=name, help=help,
                              metavar=meta, *flagNames)
        elif paramType == ONE_PARAM:
            parser.add_option(dest=name, help=help, metavar=meta, *flagNames)
        elif paramType == OPT_PARAM:
            parser.add_option(action='callback',
                               callback=optParamCallback, dest=name,
                               type='string', nargs=0, help=help,
                               metavar=meta, *flagNames)
        elif paramType == MULT_PARAM:
            parser.add_option(action='append', dest=name, help=help,
                              metavar=meta, *flagNames)
        elif paramType == COUNT_PARAM:
            parser.add_option(action='count',
                              dest=name, help=help, metavar=meta,
                              *flagNames)

def processArgs(argDef, cfgMap, cfg, usage, argv=sys.argv):
    """Mostly backwards-compatible (with earlier conary processArgs)
       function that uses optparse as its backend.
    """
    return _processArgs(argDef, cfgMap, cfg, usage, argv)[:2]

def _processArgs(params, cfgMap, cfg, usage, argv=sys.argv, version=None,
                commonParams=None, useHelp=False, defaultGroup=None,
                interspersedArgs=True, hobbleShortOpts=False):
    argSet = {}
    # don't mangle the command line
    argv = argv[:]

    # historically, usage was generally a function to print out the usage
    # message.  We want it to be a string.  For now, we
    # convert here to allow backwards compatibility.
    if hasattr(usage, '__call__'):
        stdout = StringIO.StringIO()
        stderr = StringIO.StringIO()
        oldStdOut = sys.stdout
        oldStdErr = sys.stderr
        sys.stdout = stdout
        sys.stderr = stderr
        try:
            usage()
        except SystemExit:
            # some of these old usage functions even exit after
            # printing the usage message!
            pass
        sys.stdout = oldStdOut
        sys.stderr = oldStdErr
        usage = stdout.getvalue() + stderr.getvalue()

    if defaultGroup:
        d = params[defaultGroup]
    else:
        d = params
    d['debug'] = OPT_PARAM, 'Print helpful debugging output (use --debug=all for internal debug info)'
    d['debugger'] = (NO_PARAM, optparse.SUPPRESS_HELP)

    for (arg, name) in cfgMap.items():
        d[arg] = ONE_PARAM

    parser = getOptionParser(params, cfgMap, cfg, usage, version, useHelp,
                             defaultGroup, interspersedArgs,
                             hobbleShortOpts=hobbleShortOpts)
    argSet, otherArgs, options = getArgSet(params, parser, argv)

    configFileList = argSet.pop('config-file', [])
    if not isinstance(configFileList, list):
        configFileList = [configFileList]
    for path in configFileList:
        try:
            cfg.read(path, exception = True)
        except IOError, msg:
            raise OptionError(msg, parser)

    for (arg, name) in cfgMap.items():
	if argSet.has_key(arg):
	    cfg.configLine("%s %s" % (name, argSet[arg]))
	    del argSet[arg]

    if argSet.has_key('debugger'):
	del argSet['debugger']
	from conary.lib import debugger
	debugger.set_trace()
        sys.excepthook = util.genExcepthook(cfg.dumpStackOnError,
                                            debugCtrlC=True)

    if 'debug' in argSet:
        if argSet['debug'] is True:
            log.setVerbosity(log.DEBUG)
        else:
            log.setVerbosity(log.LOWLEVEL)
	del argSet['debug']
    else:
	log.setVerbosity(log.WARNING)

    return argSet, otherArgs, parser, options

def getOptionParser(params, cfgMap, cfg, usage, version=None, useHelp=False,
                    defaultGroup=None, interspersedArgs=True,
                    hobbleShortOpts=False):
    parser = OptionParser(usage=usage, add_help_option=useHelp, version=version,
                          hobbleShortOpts=hobbleShortOpts)
    if not interspersedArgs:
        parser.disable_interspersed_args()

    if defaultGroup in params:
        group = optparse.OptionGroup(parser, defaultGroup)
        addOptions(group, params[defaultGroup])
        parser.add_option_group(group)

    found = None
    for name, data in params.iteritems():
        if name == defaultGroup:
            continue
        if isinstance(data, dict):
            found = True
            break
        else:
            found = False

    if found is False:
        group = optparse.OptionGroup(parser, 'Command Options')
        addOptions(group, params, skip=defaultGroup)
        parser.add_option_group(group)
    else:
        addOptions(parser, params, skip=defaultGroup)

    return parser

def getArgSet(params, parser, argv=sys.argv):
    (options, otherArgs) = parser.parse_args(argv)

    argSet = {}

    for name, data in params.iteritems():
        if isinstance(data, dict):
            for name in data:
                val = getattr(options, name)
                if val is None:
                    continue
                argSet[name] = val
        else:
            val = getattr(options, name)
            if val is None:
                continue
            argSet[name] = val
    return argSet, otherArgs, options

class AbstractCommand(object):
    """
        Abstract command object to be subclassed used to represent commands
        in a command line interface.  To be used with MainHandler below.
        Assumes use of a lib.cfg.ConfigFile type configuration object.
    """
    commands = []
    paramHelp = '' # for each command will display <command> + paramHelp
                   # as part of usage.
    defaultGroup = 'Common Options' # The heading for options that aren't
                                    # put in any other group.

    docs = {} # add docs in the form 'long-option' : 'description'
              # or 'long-option' : ('description', 'KEYWORD').

    def __init__(self):
        self.parser = None

    def usage(self, errNo=1):
        if self.parser:
            self.parser.print_help()
        return errNo

    def setParser(self, parser):
        self.parser = parser

    def addParameters(self, argDef):
        pass

    def addConfigOptions(self, cfgMap, argDef):
        for name, (cfgName, paramType)  in cfgMap.items():
            # if it's a NO_PARAM
            if paramType == NO_PARAM:
                negName = 'no-' + name
                argDef[self.defaultGroup][negName] = NO_PARAM, optparse.SUPPRESS_HELP
                cfgMap[negName] = (cfgName, paramType)

            argDef[self.defaultGroup][name] = paramType

    def addDocs(self, argDef):
        """ Parse a docs dict assigned at the class level
            and add those docs to the parameters being sent to
            parseOptions.
        """
        d = {}
        for class_ in reversed(inspect.getmro(self.__class__)):
            if not hasattr(class_, 'docs'):
                continue
            d.update(class_.docs)

        commandDicts = [argDef]
        while commandDicts:
            commandDict = commandDicts.pop()
            for name, value in commandDict.items():
                if isinstance(value, dict):
                    commandDicts.append(value)
                    continue
                if name in d:
                    if not isinstance(value, (list, tuple)):
                        value = [ value ]
                    else:
                        value = list(value)
                    value.append(d[name])
                    commandDict[name] = value

    def prepare(self):
        params = {}
        cfgMap = {}
        self.addParameters(params)
        self.addConfigOptions(cfgMap, params)
        self.addDocs(params)
        return params, cfgMap

    def processConfigOptions(self, cfg, cfgMap, argSet):
        """
            Manage any config maps we've set up, converting
            assigning them to the config object.
        """
        configFileList = argSet.pop('config-file', [])
        if not isinstance(configFileList, list):
            configFileList = list(configFileList)

        for line in configFileList:
            cfg.read(path, exception=True)

        for (arg, (name, paramType)) in cfgMap.items():
            value = argSet.pop(arg, None)
            if value is not None:
                if arg.startswith('no-'):
                    value = not value

                cfg.configLine("%s %s" % (name, value))

        for line in argSet.pop('config', []):
            cfg.configLine(line)

    def runCommand(self, *args, **kw):
        raise NotImplementedError

class MainHandler(object):
    """
        Class to handle parsing and executing commands set up to use
        AbstractCommands
    """

    abstractCommand = None   # class to grab generic options from.  These
                             # can be used in front of
    commandList = []         # list of commands to support.
    name = None              # name to use when showing usage messages.
    version = '<no version>' # version to return to --version

    hobbleShortOpts = False # whether or not to allow -mn to be used, or to
                            # require -m -n.
    configClass = None

    def __init__(self):
        self._supportedCommands = {}
        for command in self.commandList:
            self._registerCommand(command)

    def _registerCommand(self, commandClass):
        supportedCommands = self._supportedCommands
        inst = commandClass()
        if isinstance(commandClass.commands, str):
            supportedCommands[commandClass.commands] = inst
        else:
            for cmdName in commandClass.commands:
                supportedCommands[cmdName] = inst

    def _getPreCommandOptions(self, argv, cfg):
        """Allow the user to specify generic flags before they specify the
           command to run.
        """
        thisCommand = self.abstractCommand()
        params, cfgMap = thisCommand.prepare()
        defaultGroup = thisCommand.defaultGroup
        argSet, otherArgs, parser, optionSet = _processArgs(
                                                    params, {}, cfg,
                                                    self.usage,
                                                    argv=argv[1:],
                                                    version=self.version,
                                                    useHelp=True,
                                                    defaultGroup=defaultGroup,
                                                    interspersedArgs=False,
                                    hobbleShortOpts=self.hobbleShortOpts)
        return argSet, [argv[0]] + otherArgs

    def getConfigFile(self, argv):
        """
            Find the appropriate config file
        """
        if not self.configClass:
            raise RuntimeError, ('Must define a configClass to use with this'
                                 ' main handler')
        if '--skip-default-config' in argv:
            argv.remove('--skip-default-config')
            ccfg = self.configClass(readConfigFiles=False)
        else:
            ccfg = self.configClass(readConfigFiles=True)
        return ccfg

    def main(self, argv=sys.argv, debuggerException=Exception,
             cfg=None, **kw):
        """
            Process argv and execute commands as specified.
        """

        from conary import versions
        supportedCommands = self._supportedCommands

        if cfg is None:
            cfg = self.getConfigFile(argv)

        if '--version' in argv or '-v' in argv:
            print self.version
            return

        try:
            argSet, argv = self._getPreCommandOptions(argv, cfg)
        except debuggerException:
            raise
        except OptionError, e:
            self.usage()
            print >>sys.stderr, e
            sys.exit(e.val)

        if len(argv) < 2:
            # no command specified
            return self.usage()

        commandName = argv[1]
        if commandName not in self._supportedCommands:
            return self.usage()

        thisCommand = self._supportedCommands[commandName]
        params, cfgMap = thisCommand.prepare()
        defaultGroup = thisCommand.defaultGroup
        if self.name:
            progName = self.name
        else:
            progName = argv[0]
        commandUsage = '%s %s %s' % (progName, commandName,
                                     thisCommand.paramHelp)
        try:
            newArgSet, otherArgs, parser, optionSet = _processArgs(
                                        params, {}, cfg,
                                        commandUsage,
                                        argv=argv,
                                        version=self.version,
                                        useHelp=True,
                                        defaultGroup=defaultGroup,
                                        hobbleShortOpts=self.hobbleShortOpts)
        except debuggerException, e:
            raise
        except OptionError, e:
            e.parser.print_help()
            print >> sys.stderr, e
            sys.exit(e.val)
        except versions.ParseError, e:
            print >> sys.stderr, e
            sys.exit(1)

        thisCommand.setParser(parser)
        argSet.update(newArgSet)
        thisCommand.processConfigOptions(cfg, cfgMap, argSet)
        self.runCommand(thisCommand, cfg, argSet, otherArgs, **kw)

    def runCommand(self, thisCommand, *args, **kw):
        thisCommand.runCommand(*args, **kw)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents

r.connect(host='rdb').repl()
v = r.db('webapp')
a = v.table('datadocs').update({"type":'documents'}).run()

"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi.resources.utilities import split_and_html_strip
from restapi.resources.custom.docs import image_destination
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
# TESTING = False
TESTING = True

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = 'datadmins'

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()

#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_query()
        # test_el()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    # remove pending files...
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################

def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        elif len(obj['record']) > 0:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)
    # REMOVE

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({
            pkey: record,
            'images': images,
            'type': image_destination({})
        }).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    # # Elasticsearch magic
    # print("Elasticsearch and indexes")
    # es = Elasticsearch(hosts=[ES_HOST])
    # es.indices.delete(index=EL_INDEX, ignore=[400, 404])
    # es.indices.create(index=EL_INDEX, ignore=400)
    # #es.indices.refresh(index=EL_INDEX)

    for record, rows in res.items():
        steps = []
        title = None

        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []

                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        if field['position'] == 1:
                            title = value
                        break

                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

# PLUG ELASTICSEARCH SOMEWHERE IN HERE
        doc = {
            'category': STEPS[row['step']],
            'text': title,
            'timestamp': datetime.now(),
        }
# # TO FIX?
#         #print(doc)
#         es.index(index=EL_INDEX, doc_type='normal', body=doc)
#         #print("DEBUG EXIT"); exit(1)

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")
        tmp = new['step']
        STEPS[tmp['num']] = tmp['name']
    print(STEPS)

def test_query():
    """ test queries on rdb """

    document = '5fc8d3f4-59ee-43ca-9543-6241bb820882'

    extra = {
        'data': [
            {'value': 'Test', 'name': 'Personnages', 'position': 1, 'hash': '035ca5c7'},
            {'value': 'From paolo', 'name': 'Artistes', 'position': 2, 'hash': '01b1020a'}
        ],
        'latest_db_info':
            {'timestamp': 1458124102.565326, 'user_id': '1a33400a', 'ip': '109.89.122.137'},
        'step': 4
    }

    q = query.get_table_query(t2in)
    cursor = q.get(document).run()
    cursor['steps'].append(extra)
    print(cursor, type(cursor))

    changes = q.get(document).replace(cursor).run()
    #     .update(lambda row: row['steps'].append(extra))
    print(changes)
    # print(cursor.run())
    # print(list(cursor.run()))
    print("DEBUG"); exit(1)

    ##################################

    q = query.get_table_query(t4in)
    cursor = q \
        .filter({'type': 'welcome'}).without('type') \
        .eq_join("id", r.table(t3in), index="record") \
        .zip() \
        .filter({'type': 'welcome'})
    print(list(cursor.run()))
    print("DEBUG"); exit(1)

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def test_el():
    print("TEST")
    es = Elasticsearch(hosts=[ES_HOST])
    print(es)

    # doc = {
    #     'author': 'kimchy',
    #     'text': 'Elasticsearch: cool. bonsai cool.',
    #     'timestamp': datetime.now(),
    # }
    # res = es.index(index="test-index", doc_type='tweet', id=1, body=doc)
    # print(res['created'])

    # Note:
    # Refresh indices at login startup!!!
    es.indices.refresh(index="test-index")

    res = es.get(index="test-index", doc_type='tweet', id=1)
    print(res['_source'])

    #es.search(index='posts', q='author:"Benjamin Pollack"')

    exit(1)
<commit_msg>Testing rethinkdb second indexing<commit_after># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents

r.connect(host='rdb').repl()
v = r.db('webapp')
a = v.table('datadocs').update({"type":'documents'}).run()

"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi.resources.utilities import split_and_html_strip
from restapi.resources.custom.docs import image_destination
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
# TESTING = False
TESTING = True

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = 'datadmins'

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()

#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_query()
        # test_el()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    # remove pending files...
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################

def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        elif len(obj['record']) > 0:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)
    # REMOVE

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({
            pkey: record,
            'images': images,
            'type': image_destination({})
        }).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    # # Elasticsearch magic
    # print("Elasticsearch and indexes")
    # es = Elasticsearch(hosts=[ES_HOST])
    # es.indices.delete(index=EL_INDEX, ignore=[400, 404])
    # es.indices.create(index=EL_INDEX, ignore=400)
    # #es.indices.refresh(index=EL_INDEX)

    for record, rows in res.items():
        steps = []
        title = None

        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []

                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        if field['position'] == 1:
                            title = value
                        break

                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

# PLUG ELASTICSEARCH SOMEWHERE IN HERE
        doc = {
            'category': STEPS[row['step']],
            'text': title,
            'timestamp': datetime.now(),
        }
# # TO FIX?
#         #print(doc)
#         es.index(index=EL_INDEX, doc_type='normal', body=doc)
#         #print("DEBUG EXIT"); exit(1)

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")
        tmp = new['step']
        STEPS[tmp['num']] = tmp['name']
    print(STEPS)

def test_query():
    """ test queries on rdb """

    ###################################################
    # Try optimization with seconday index
    q = query.get_table_query(t2in)

    index = "title"
    search = "Limoges_33"

    if index in q.index_list().run():
        print("Dropping")
        q.index_drop(index).run()
    print("Creating")
    q.index_create(index, lambda doc: doc['steps'].map(
        lambda step: step['data'].map(lambda data: data['value'])),
        multi=True) \
        .run()
    print("Waiting")
    q.index_wait(index).run()
    print("Done")

    # print("Status", q.index_status().run())
    cursor = q.get_all(search, index=index).run()
    print("TEST", list(cursor))
    exit(1)

    ###################################################
    document = '5fc8d3f4-59ee-43ca-9543-6241bb820882'
    extra = {
        'data': [
            {'value': 'Test', 'name': 'Personnages', 'position': 1, 'hash': '035ca5c7'},
            {'value': 'From paolo', 'name': 'Artistes', 'position': 2, 'hash': '01b1020a'}
        ],
        'latest_db_info':
            {'timestamp': 1458124102.565326, 'user_id': '1a33400a', 'ip': '109.89.122.137'},
        'step': 4
    }

    q = query.get_table_query(t2in)
    cursor = q.get(document).run()
    cursor['steps'].append(extra)
    print(cursor, type(cursor))

    changes = q.get(document).replace(cursor).run()
    #     .update(lambda row: row['steps'].append(extra))
    print(changes)
    # print(cursor.run())
    # print(list(cursor.run()))
    print("DEBUG"); exit(1)

    ##################################

    q = query.get_table_query(t4in)
    cursor = q \
        .filter({'type': 'welcome'}).without('type') \
        .eq_join("id", r.table(t3in), index="record") \
        .zip() \
        .filter({'type': 'welcome'})
    print(list(cursor.run()))
    print("DEBUG"); exit(1)

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def test_el():
    print("TEST")
    es = Elasticsearch(hosts=[ES_HOST])
    print(es)

    # doc = {
    #     'author': 'kimchy',
    #     'text': 'Elasticsearch: cool. bonsai cool.',
    #     'timestamp': datetime.now(),
    # }
    # res = es.index(index="test-index", doc_type='tweet', id=1, body=doc)
    # print(res['created'])

    # Note:
    # Refresh indices at login startup!!!
    es.indices.refresh(index="test-index")

    res = es.get(index="test-index", doc_type='tweet', id=1)
    print(res['_source'])

    #es.search(index='posts', q='author:"Benjamin Pollack"')

    exit(1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
TESTING = False

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = "datapending"
t5in = "datamissing"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()
    if t4in in tables:
        query.get_query().table_drop(t4in).run()

#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_el()
        test_query()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    ##if t4in not in tables:
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################

def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        else:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)

    # q = query.get_table_query(t4in)
    # q = query.get_table_query(t5in)
    # REMOVE

def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({pkey: record, 'images': images}).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    # # Elasticsearch magic
    # print("Elasticsearch and indexes")
    # es = Elasticsearch(hosts=[ES_HOST])
    # es.indices.delete(index=EL_INDEX, ignore=[400, 404])
    # es.indices.create(index=EL_INDEX, ignore=400)
    # #es.indices.refresh(index=EL_INDEX)

    for record, rows in res.items():
        steps = []
        title = None

        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []

                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        if field['position'] == 1:
                            title = value
                        break

                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

# PLUG ELASTICSEARCH SOMEWHERE IN HERE
        doc = {
            'category': STEPS[row['step']],
            'text': title,
            'timestamp': datetime.now(),
        }
# # TO FIX?
#         #print(doc)
#         es.index(index=EL_INDEX, doc_type='normal', body=doc)
#         #print("DEBUG EXIT"); exit(1)

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")
        tmp = new['step']
        STEPS[tmp['num']] = tmp['name']
    print(STEPS)

def test_query():
    """ test queries on rdb """
    # q = query.get_table_query(t2in)
    q = query.get_table_query(t3in)

    # cursor = q \
    #     .concat_map(
    #         lambda doc:
    #             doc['images'].has_fields({'transcriptions': True}).concat_map(
    #                 lambda image: image['transcriptions_split'])) \
    #     .distinct() \
    #     .run()

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

# #TEST1
#     cursor = q \
#         .concat_map(
#             lambda x: x['steps']['data'].map(
#                 lambda item: item['value'])
#         ) \
#         .run()
#     for obj in cursor:
#         print("TEST", obj)
#         exit(1)
#     print(list(cursor))
#     exit(1)

#WORKING FOR RECOVERING DATA
    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def test_el():
    print("TEST")
    es = Elasticsearch(hosts=[ES_HOST])
    print(es)

    # doc = {
    #     'author': 'kimchy',
    #     'text': 'Elasticsearch: cool. bonsai cool.',
    #     'timestamp': datetime.now(),
    # }
    # res = es.index(index="test-index", doc_type='tweet', id=1, body=doc)
    # print(res['created'])

    # Note:
    # Refresh indices at login startup!!!
    es.indices.refresh(index="test-index")

    res = es.get(index="test-index", doc_type='tweet', id=1)
    print(res['_source'])

    #es.search(index='posts', q='author:"Benjamin Pollack"')

    exit(1)
<commit_msg>Testing how to check for empty data<commit_after># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
#TESTING = False
TESTING = True

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = "datapending"
t5in = "datamissing"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()
    if t4in in tables:
        query.get_query().table_drop(t4in).run()

#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_query()
        # test_el()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    ##if t4in not in tables:
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################

def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        else:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)

    # q = query.get_table_query(t4in)
    # q = query.get_table_query(t5in)
    # REMOVE

def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({pkey: record, 'images': images}).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    # # Elasticsearch magic
    # print("Elasticsearch and indexes")
    # es = Elasticsearch(hosts=[ES_HOST])
    # es.indices.delete(index=EL_INDEX, ignore=[400, 404])
    # es.indices.create(index=EL_INDEX, ignore=400)
    # #es.indices.refresh(index=EL_INDEX)

    for record, rows in res.items():
        steps = []
        title = None

        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []

                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        if field['position'] == 1:
                            title = value
                        break

                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

# PLUG ELASTICSEARCH SOMEWHERE IN HERE
        doc = {
            'category': STEPS[row['step']],
            'text': title,
            'timestamp': datetime.now(),
        }
# # TO FIX?
#         #print(doc)
#         es.index(index=EL_INDEX, doc_type='normal', body=doc)
#         #print("DEBUG EXIT"); exit(1)

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")
        tmp = new['step']
        STEPS[tmp['num']] = tmp['name']
    print(STEPS)

def test_query():
    """ test queries on rdb """

    q = query.get_table_query(t3in)

    cursor = q. \
        filter({'record': '0a98cdbf-4db6-4042-812b-033ca9dda009'})['images'] \
        .run()
    print(next(cursor))

    print("DEBUG")
    exit(1)

    # cursor = q \
    #     .concat_map(
    #         lambda doc:
    #             doc['images'].has_fields({'transcriptions': True}).concat_map(
    #                 lambda image: image['transcriptions_split'])) \
    #     .distinct() \
    #     .run()

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

# #TEST1
#     cursor = q \
#         .concat_map(
#             lambda x: x['steps']['data'].map(
#                 lambda item: item['value'])
#         ) \
#         .run()
#     for obj in cursor:
#         print("TEST", obj)
#         exit(1)
#     print(list(cursor))
#     exit(1)

#WORKING FOR RECOVERING DATA
    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def test_el():
    print("TEST")
    es = Elasticsearch(hosts=[ES_HOST])
    print(es)

    # doc = {
    #     'author': 'kimchy',
    #     'text': 'Elasticsearch: cool. bonsai cool.',
    #     'timestamp': datetime.now(),
    # }
    # res = es.index(index="test-index", doc_type='tweet', id=1, body=doc)
    # print(res['created'])

    # Note:
    # Refresh indices at login startup!!!
    es.indices.refresh(index="test-index")

    res = es.get(index="test-index", doc_type='tweet', id=1)
    print(res['_source'])

    #es.search(index='posts', q='author:"Benjamin Pollack"')

    exit(1)
<|endoftext|>