<commit_before>#!/usr/bin/env python
#
# A script for unpacking and installing different historic versions of
# SCons in a consistent manner for side-by-side development testing.
#
# This abstracts the changes we've made to the SCons setup.py scripts in
# different versions so that, no matter what version is specified, it ends
# up installing the necessary script(s) and library into version-specific
# names that won't interfere with other things.
#
# By default, we expect to extract the .tar.gz files from a Downloads
# subdirectory in the current directory.
#
# Note that this script cleans up after itself, removing the extracted
# directory in which we do the build.
#
# This was written for a Linux system (specifically Ubuntu) but should
# be reasonably generic to any POSIX-style system with a /usr/local
# hierarchy.

import getopt
import os
import shutil
import sys

from Command import CommandRunner, Usage

all_versions = [
    '0.01',
    '0.02',
    '0.03',
    '0.04',
    '0.05',
    '0.06',
    '0.07',
    '0.08',
    '0.09',
    '0.10',
    '0.11',
    '0.12',
    '0.13',
    '0.14',
    '0.90',
    '0.91',
    '0.92',
    '0.93',
    '0.94',
    #'0.94.1',
    '0.95',
    #'0.95.1',
    '0.96',
    '0.96.1',
    '0.96.90',
    '0.96.91',
    '0.96.92',
    '0.96.93',
    '0.96.94',
    '0.96.95',
    '0.96.96',
    '0.97',
    '0.97.0d20070809',
    '0.97.0d20070918',
    '0.97.0d20071212',
    '0.98.0',
    '0.98.1',
    '0.98.2',
    '0.98.3',
    '0.98.4',
    '0.98.5',
    '1.0.0',
    '1.0.0.d20080826',
    '1.0.1',
    '1.0.1.d20080915',
    '1.0.1.d20081001',
]

def main(argv=None):
    if argv is None:
        argv = sys.argv

    all = False
    downloads_dir = 'Downloads'
    downloads_url = 'http://downloads.sourceforge.net/scons'
    sudo = 'sudo'
    prefix = '/usr/local'
    python = sys.executable

    short_options = 'ad:hnp:q'
    long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet']

    helpstr = """\
Usage:  install-scons.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...]

  -a, --all                     Install all SCons versions.
  -d DIR, --downloads=DIR       Downloads directory.
  -h, --help                    Print this help and exit
  -n, --no-exec                 No execute, just print the command line
  -p PREFIX, --prefix=PREFIX    Installation prefix.
  -q, --quiet                   Quiet, don't print the command line
"""

    try:
        try:
            opts, args = getopt.getopt(argv[1:], short_options, long_options)
        except getopt.error, msg:
            raise Usage(msg)

        for o, a in opts:
            if o in ('-a', '--all'):
                all = True
            elif o in ('-d', '--downloads'):
                downloads_dir = a
            elif o in ('-h', '--help'):
                print helpstr
                sys.exit(0)
            elif o in ('-n', '--no-exec'):
                CommandRunner.execute = CommandRunner.do_not_execute
            elif o in ('-p', '--prefix'):
                prefix = a
            elif o in ('-q', '--quiet'):
                CommandRunner.display = CommandRunner.do_not_display
    except Usage, err:
        sys.stderr.write(str(err.msg) + '\n')
        sys.stderr.write('use -h to get help\n')
        return 2

    if all:
        if args:
            msg = 'install-scons.py:  -a and version arguments both specified'
            sys.stderr.write(msg)
            sys.exit(1)

        args = all_versions

    cmd = CommandRunner()

    for version in args:
        scons = 'scons-' + version
        tar_gz = os.path.join(downloads_dir, scons + '.tar.gz')
        tar_gz_url = os.path.join(downloads_url, scons + '.tar.gz')

        cmd.subst_dictionary(locals())

        if not os.path.exists(tar_gz):
            if not os.path.exists(downloads_dir):
                cmd.run((os.mkdir, downloads_dir),
                        'mkdir %(downloads_dir)s')
            cmd.run('wget -O %(tar_gz)s %(tar_gz_url)s')

        cmd.run('tar zxf %(tar_gz)s')

        cmd.run((os.chdir, scons), 'cd %(scons)s')

        if version in ('0.01', '0.02', '0.03', '0.04', '0.05',
                       '0.06', '0.07', '0.08', '0.09', '0.10'):

            # 0.01 through 0.10 install /usr/local/bin/scons and
            # /usr/local/lib/scons.  The "scons" script knows how to
            # look up the library in a version-specific directory, but
            # we have to move both it and the library directory into
            # the right version-specific name by hand.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')
            cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s')

        elif version in ('0.11', '0.12', '0.13', '0.14', '0.90'):

            # 0.11 through 0.90 install /usr/local/bin/scons and
            # /usr/local/lib/scons-%(version)s.  We just need to move
            # the script to a version-specific name.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')

        elif version in ('0.91', '0.92', '0.93',
                         '0.94', '0.94.1',
                         '0.95', '0.95.1',
                         '0.96', '0.96.1', '0.96.90'):

            # 0.91 through 0.96.90 install /usr/local/bin/scons,
            # /usr/local/bin/sconsign and /usr/local/lib/scons-%(version)s.
            # We need to move both scripts to version-specific names.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/sconsign %(prefix)s/bin/sconsign-%(version)s')
            lib_scons = os.path.join(prefix, 'lib', 'scons')
            if os.path.isdir(lib_scons):
                cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s')

        else:

            # Versions from 0.96.91 and later support what we want
            # with a --no-scons-script option.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s --no-scons-script')

        cmd.run((os.chdir, '..'), 'cd ..')

        cmd.run((shutil.rmtree, scons), 'rm -rf %(scons)s')

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Add 1.1.0 to the instal_scons.py script.<commit_after>#!/usr/bin/env python
#
# A script for unpacking and installing different historic versions of
# SCons in a consistent manner for side-by-side development testing.
#
# This abstracts the changes we've made to the SCons setup.py scripts in
# different versions so that, no matter what version is specified, it ends
# up installing the necessary script(s) and library into version-specific
# names that won't interfere with other things.
#
# By default, we expect to extract the .tar.gz files from a Downloads
# subdirectory in the current directory.
#
# Note that this script cleans up after itself, removing the extracted
# directory in which we do the build.
#
# This was written for a Linux system (specifically Ubuntu) but should
# be reasonably generic to any POSIX-style system with a /usr/local
# hierarchy.

import getopt
import os
import shutil
import sys

from Command import CommandRunner, Usage

all_versions = [
    '0.01',
    '0.02',
    '0.03',
    '0.04',
    '0.05',
    '0.06',
    '0.07',
    '0.08',
    '0.09',
    '0.10',
    '0.11',
    '0.12',
    '0.13',
    '0.14',
    '0.90',
    '0.91',
    '0.92',
    '0.93',
    '0.94',
    #'0.94.1',
    '0.95',
    #'0.95.1',
    '0.96',
    '0.96.1',
    '0.96.90',
    '0.96.91',
    '0.96.92',
    '0.96.93',
    '0.96.94',
    '0.96.95',
    '0.96.96',
    '0.97',
    '0.97.0d20070809',
    '0.97.0d20070918',
    '0.97.0d20071212',
    '0.98.0',
    '0.98.1',
    '0.98.2',
    '0.98.3',
    '0.98.4',
    '0.98.5',
    '1.0.0',
    '1.0.0.d20080826',
    '1.0.1',
    '1.0.1.d20080915',
    '1.0.1.d20081001',
    '1.1.0',
]

def main(argv=None):
    if argv is None:
        argv = sys.argv

    all = False
    downloads_dir = 'Downloads'
    downloads_url = 'http://downloads.sourceforge.net/scons'
    sudo = 'sudo'
    prefix = '/usr/local'
    python = sys.executable

    short_options = 'ad:hnp:q'
    long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet']

    helpstr = """\
Usage:  install-scons.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...]

  -a, --all                     Install all SCons versions.
  -d DIR, --downloads=DIR       Downloads directory.
  -h, --help                    Print this help and exit
  -n, --no-exec                 No execute, just print the command line
  -p PREFIX, --prefix=PREFIX    Installation prefix.
  -q, --quiet                   Quiet, don't print the command line
"""

    try:
        try:
            opts, args = getopt.getopt(argv[1:], short_options, long_options)
        except getopt.error, msg:
            raise Usage(msg)

        for o, a in opts:
            if o in ('-a', '--all'):
                all = True
            elif o in ('-d', '--downloads'):
                downloads_dir = a
            elif o in ('-h', '--help'):
                print helpstr
                sys.exit(0)
            elif o in ('-n', '--no-exec'):
                CommandRunner.execute = CommandRunner.do_not_execute
            elif o in ('-p', '--prefix'):
                prefix = a
            elif o in ('-q', '--quiet'):
                CommandRunner.display = CommandRunner.do_not_display
    except Usage, err:
        sys.stderr.write(str(err.msg) + '\n')
        sys.stderr.write('use -h to get help\n')
        return 2

    if all:
        if args:
            msg = 'install-scons.py:  -a and version arguments both specified'
            sys.stderr.write(msg)
            sys.exit(1)

        args = all_versions

    cmd = CommandRunner()

    for version in args:
        scons = 'scons-' + version
        tar_gz = os.path.join(downloads_dir, scons + '.tar.gz')
        tar_gz_url = os.path.join(downloads_url, scons + '.tar.gz')

        cmd.subst_dictionary(locals())

        if not os.path.exists(tar_gz):
            if not os.path.exists(downloads_dir):
                cmd.run((os.mkdir, downloads_dir),
                        'mkdir %(downloads_dir)s')
            cmd.run('wget -O %(tar_gz)s %(tar_gz_url)s')

        cmd.run('tar zxf %(tar_gz)s')

        cmd.run((os.chdir, scons), 'cd %(scons)s')

        if version in ('0.01', '0.02', '0.03', '0.04', '0.05',
                       '0.06', '0.07', '0.08', '0.09', '0.10'):

            # 0.01 through 0.10 install /usr/local/bin/scons and
            # /usr/local/lib/scons.  The "scons" script knows how to
            # look up the library in a version-specific directory, but
            # we have to move both it and the library directory into
            # the right version-specific name by hand.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')
            cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s')

        elif version in ('0.11', '0.12', '0.13', '0.14', '0.90'):

            # 0.11 through 0.90 install /usr/local/bin/scons and
            # /usr/local/lib/scons-%(version)s.  We just need to move
            # the script to a version-specific name.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')

        elif version in ('0.91', '0.92', '0.93',
                         '0.94', '0.94.1',
                         '0.95', '0.95.1',
                         '0.96', '0.96.1', '0.96.90'):

            # 0.91 through 0.96.90 install /usr/local/bin/scons,
            # /usr/local/bin/sconsign and /usr/local/lib/scons-%(version)s.
            # We need to move both scripts to version-specific names.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s')
            cmd.run('%(sudo)s mv %(prefix)s/bin/sconsign %(prefix)s/bin/sconsign-%(version)s')
            lib_scons = os.path.join(prefix, 'lib', 'scons')
            if os.path.isdir(lib_scons):
                cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s')

        else:

            # Versions from 0.96.91 and later support what we want
            # with a --no-scons-script option.
            cmd.run('%(python)s setup.py build')
            cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s --no-scons-script')

        cmd.run((os.chdir, '..'), 'cd ..')

        cmd.run((shutil.rmtree, scons), 'rm -rf %(scons)s')

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>
import dis
from FuncModify import restart_func

def _find_traceframe(tb, code):
	while tb:
		if tb.tb_frame.f_code is code: return tb
		tb = tb.tb_next
	return None

def demo1():
	def demoFunc(a,b,c, raiseExc=None):
		print("a: %r" % a)
		while b > 0:
			print("b: %r" % b)
			if b == 4 and raiseExc: raise raiseExc
			if b == 2:
				b = 0
				continue
			b -= 1
		print("c: %r" % c)

	func = demoFunc
	try:
		# This prints:
		#   a: 'start'
		#   b: 5
		# And throws the exception then.
		demoFunc("start", 5, "end", Exception)
		assert False
	except Exception:
		print "! Exception"
		import sys
		_,_,tb = sys.exc_info()

	tb = _find_traceframe(tb, func.func_code)
	assert tb is not None

	# Start just one after the `raise`.
	instraddr = min([addr for (addr,_) in dis.findlinestarts(func.func_code) if addr > tb.tb_lasti])
	# Play around. Avoid that we throw the exception again.
	localdict = dict(tb.tb_frame.f_locals)
	localdict["b"] = 5
	localdict["raiseExc"] = None
	new_func = restart_func(func, instraddr=instraddr, localdict=localdict)

	# This prints:
	#   b: 4
	#   b: 3
	#   b: 2
	#   c: 'end'
	new_func()

def _calc_newlineno_via_diff(oldlineno, oldfilename, newfilename):
	from subprocess import Popen, PIPE
	diffcmd = ["diff", "-EbwB", "-U", "0"]
	diffout = Popen(diffcmd + [oldfilename, newfilename], stdout=PIPE).stdout.readlines()
	lineno = 1
	newlineno = 1
	import re
	r = re.compile(r"^@@ -([0-9]+).*@@.*$")
	for diffline in diffout:
		if diffline.startswith("--- "): continue
		if diffline.startswith("+++ "): continue
		m = r.match(diffline)
		if m:
			nextlineno = int(m.groups()[0])
			if nextlineno > oldlineno:
				return newlineno + (oldlineno - lineno)
			newlineno += (nextlineno - lineno)
			lineno = nextlineno
			continue
		assert diffline[0:1] in "+-" # because of "diff -U 0"
		if diffline[0] == "+":
			newlineno += 1
		elif diffline[0] == "-":
			lineno += 1
		if lineno >= oldlineno:
			return newlineno
	return newlineno

def demo2():
	# Let the user create some Python code.
	import os, tempfile
	editcmd = os.environ.get("EDITOR", "vi")
	tmpfn = tempfile.mktemp(suffix=".py")
	open(tmpfn, "w").write(
		"def main():\n"
		"	# Write some code here.\n"
		"	# When done, safe and exit. This code will get executed then.\n"
		"	# You can raise an exception via code here.\n"
		"	# Or just write an infinite loop and press Ctrl+C at execution.\n"
		"	import time\n"
		"	i = 0\n"
		"	while True:\n"
		"		i += 1\n"
		"		print i\n"
		"		time.sleep(1)\n"
		"\n"
	)
	os.system("%r %r" % (editcmd, tmpfn))

	# Make a backup of the file.
	tmpbackupfn = tempfile.mktemp(suffix=".py")
	os.system("cp %r %r" % (tmpfn, tmpbackupfn))

	# Now run the code.
	import imp
	tmpfnmod = imp.load_source("tmpfnmod", tmpfn)
	func = tmpfnmod.main
	try:
		func()
		assert False, "You must raise an exception for this demo."
	except BaseException:
		import sys
		excinfo = sys.exc_info()
		sys.excepthook(*excinfo)
		tb = excinfo[-1]
	tb = _find_traceframe(tb, func.func_code)
	assert tb is not None
	localdict = dict(tb.tb_frame.f_locals)

	# Write hint about exception.
	lines = list(open(tmpfn).readlines())
	lineno = tb.tb_lineno - 1
	lines[lineno:lineno] = ["# The exception was raised in the following line!\n"]
	open(tmpfn, "w").writelines(lines)

	# Let the user edit the file again.
	os.system("%r %r" % (editcmd, tmpfn))

	# Reload file.
	tmpfnmod = imp.load_source("tmpfnmod", tmpfn)
	func = tmpfnmod.main
	# Calculate new line number.
	newlineno = _calc_newlineno_via_diff(tb.tb_lineno + 1, tmpbackupfn, tmpfn)
	print "Starting in line %i at %r" % (newlineno, open(tmpfn).readlines()[newlineno].strip("\n"))
	# Use that instruction address.
	instraddr = min([addr for (addr,lineno) in dis.findlinestarts(func.func_code) if lineno >= newlineno])

	# Register some cleanup handlers.
	import atexit
	atexit.register(lambda: os.remove(tmpfn))
	atexit.register(lambda: os.remove(tmpbackupfn))

	# Now, do the magic!
	# Create a modified function which jumps right to the previous place.
	newfunc = restart_func(func, instraddr=instraddr, localdict=localdict)
	# And run it.
	newfunc()

if __name__ == "__main__":
	import sys
	demoname = "demo1"
	if len(sys.argv) > 0: demoname = sys.argv[1]
	globals()[demoname]()
<commit_msg>small fix<commit_after>
import dis
from FuncModify import restart_func

def _find_traceframe(tb, code):
	while tb:
		if tb.tb_frame.f_code is code: return tb
		tb = tb.tb_next
	return None

def demo1():
	def demoFunc(a,b,c, raiseExc=None):
		print("a: %r" % a)
		while b > 0:
			print("b: %r" % b)
			if b == 4 and raiseExc: raise raiseExc
			if b == 2:
				b = 0
				continue
			b -= 1
		print("c: %r" % c)

	func = demoFunc
	try:
		# This prints:
		#   a: 'start'
		#   b: 5
		# And throws the exception then.
		demoFunc("start", 5, "end", Exception)
		assert False
	except Exception:
		print "! Exception"
		import sys
		_,_,tb = sys.exc_info()

	tb = _find_traceframe(tb, func.func_code)
	assert tb is not None

	# Start just one after the `raise`.
	instraddr = min([addr for (addr,_) in dis.findlinestarts(func.func_code) if addr > tb.tb_lasti])
	# Play around. Avoid that we throw the exception again.
	localdict = dict(tb.tb_frame.f_locals)
	localdict["b"] = 5
	localdict["raiseExc"] = None
	new_func = restart_func(func, instraddr=instraddr, localdict=localdict)

	# This prints:
	#   b: 4
	#   b: 3
	#   b: 2
	#   c: 'end'
	new_func()

def _calc_newlineno_via_diff(oldlineno, oldfilename, newfilename):
	from subprocess import Popen, PIPE
	diffcmd = ["diff", "-EbwB", "-U", "0"]
	diffout = Popen(diffcmd + [oldfilename, newfilename], stdout=PIPE).stdout.readlines()
	lineno = 1
	newlineno = 1
	import re
	r = re.compile(r"^@@ -([0-9]+).*@@.*$")
	for diffline in diffout:
		if diffline.startswith("--- "): continue
		if diffline.startswith("+++ "): continue
		m = r.match(diffline)
		if m:
			nextlineno = int(m.groups()[0])
			nextlineno += 1 # diff lineno's start at 0
			if nextlineno > oldlineno:
				return newlineno + (oldlineno - lineno)
			newlineno += (nextlineno - lineno)
			lineno = nextlineno
			continue
		assert diffline[0:1] in "+-" # because of "diff -U 0"
		if diffline[0] == "+":
			newlineno += 1
		elif diffline[0] == "-":
			lineno += 1
		if lineno >= oldlineno:
			return newlineno
	return newlineno

def demo2():
	# Let the user create some Python code.
	import os, tempfile
	editcmd = os.environ.get("EDITOR", "vi")
	tmpfn = tempfile.mktemp(suffix=".py")
	open(tmpfn, "w").write(
		"def main():\n"
		"	# Write some code here.\n"
		"	# When done, safe and exit. This code will get executed then.\n"
		"	# You can raise an exception via code here.\n"
		"	# Or just write an infinite loop and press Ctrl+C at execution.\n"
		"	import time\n"
		"	i = 0\n"
		"	while True:\n"
		"		i += 1\n"
		"		print i\n"
		"		time.sleep(1)\n"
		"\n"
	)
	os.system("%r %r" % (editcmd, tmpfn))

	# Make a backup of the file.
	tmpbackupfn = tempfile.mktemp(suffix=".py")
	os.system("cp %r %r" % (tmpfn, tmpbackupfn))

	# Now run the code.
	import imp
	tmpfnmod = imp.load_source("tmpfnmod", tmpfn)
	func = tmpfnmod.main
	try:
		func()
		assert False, "You must raise an exception for this demo."
	except BaseException:
		import sys
		excinfo = sys.exc_info()
		sys.excepthook(*excinfo)
		tb = excinfo[-1]
	tb = _find_traceframe(tb, func.func_code)
	assert tb is not None
	localdict = dict(tb.tb_frame.f_locals)

	# Write hint about exception.
	lines = list(open(tmpfn).readlines())
	lineno = tb.tb_lineno - 1
	lines[lineno:lineno] = ["# The exception was raised in the following line!\n"]
	open(tmpfn, "w").writelines(lines)

	# Let the user edit the file again.
	os.system("%r %r" % (editcmd, tmpfn))

	# Reload file.
	tmpfnmod = imp.load_source("tmpfnmod", tmpfn)
	func = tmpfnmod.main
	# Calculate new line number.
	newlineno = _calc_newlineno_via_diff(tb.tb_lineno, tmpbackupfn, tmpfn)
	print "Restarting in line %i at %r" % (newlineno, open(tmpfn).readlines()[newlineno-1].strip())
	# Use that instruction address.
	instraddr = min([addr for (addr,lineno) in dis.findlinestarts(func.func_code) if lineno >= newlineno])

	# Register some cleanup handlers.
	import atexit
	atexit.register(lambda: os.remove(tmpfn))
	atexit.register(lambda: os.remove(tmpbackupfn))

	# Now, do the magic!
	# Create a modified function which jumps right to the previous place.
	newfunc = restart_func(func, instraddr=instraddr, localdict=localdict)
	# And run it.
	newfunc()

if __name__ == "__main__":
	import sys
	demoname = "demo1"
	if len(sys.argv) > 0: demoname = sys.argv[1]
	globals()[demoname]()
<|endoftext|>


<commit_before>"""
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""
import sys
import os
import logging
from datetime import timedelta
from django.urls import reverse
from urllib.parse import quote
from minio import Minio
from gwells.settings.base import get_env_variable

logger = logging.getLogger(__name__)

class MinioClient():
    """ Load a minio client to handle public and/or private file requests

    Requires environment variables:
        S3_HOST: hostname for public document storage
        S3_ROOT_BUCKET: public Wells document storage bucket
        S3_AQUIFER_BUCKET: public Aquifers document storage buck
        MINIO_ACCESS_KEY: private storage account
        MINIO_SECRET_KEY: private storage secret
        S3_PRIVATE_HOST: private storage host (must be specified even if same as public storage)
        S3_PRIVATE_BUCKET: private storage bucket

    The optional "request" param can be set to the request that requires the minio client.
    This allows generation of full URIs including domain name.
    This is only required for generating private, local links.

    e.g.:
    def get(self, request):
        client = MinioClient(request)

    """

    def __init__(self, request=None, disable_public=False, disable_private=False):
        self.request = request

        if not disable_public:
            self.public_host = get_env_variable('S3_HOST', strict=True)
            self.public_bucket = get_env_variable(
                'S3_ROOT_BUCKET', strict=True)
            self.public_aquifers_bucket = get_env_variable('S3_AQUIFER_BUCKET', default_value="aquifer-docs")
            self.public_drillers_bucket = get_env_variable('S3_REGISTRANT_BUCKET', default_value="driller-docs")
            self.public_access_key = get_env_variable(
                'S3_PUBLIC_ACCESS_KEY', warn=False)
            self.public_secret_key = get_env_variable(
                'S3_PUBLIC_SECRET_KEY', warn=False)
            self.use_secure = int(get_env_variable(
                'S3_USE_SECURE', 1, warn=False))

            self.public_client = Minio(
                self.public_host,
                access_key=self.public_access_key,
                secret_key=self.public_secret_key,
                secure=self.use_secure
            )

        if not disable_private:
            self.private_client = self.create_private_client()

        self.disable_private = disable_private

    def create_private_client(self):
        self.private_access_key = get_env_variable('MINIO_ACCESS_KEY')
        self.private_secret_key = get_env_variable('MINIO_SECRET_KEY')
        self.private_host = get_env_variable('S3_PRIVATE_HOST')
        self.private_bucket = get_env_variable('S3_PRIVATE_BUCKET')
        self.private_aquifers_bucket = get_env_variable('S3_PRIVATE_AQUIFER_BUCKET', default_value="aquifer-docs")
        self.private_drillers_bucket = get_env_variable('S3_PRIVATE_REGISTRANT_BUCKET', default_value="driller-docs")

        return Minio(
            self.private_host,
            access_key=self.private_access_key,
            secret_key=self.private_secret_key,
            secure=self.use_secure
        )

    def get_private_file(self, object_name: str):
        """ Generates a link to a private document with name "object_name" (name includes prefixes) """
        return self.private_client.presigned_get_object(
            self.private_bucket,
            object_name,
            expires=timedelta(minutes=12))

    def create_url(self, obj, host, private=False):
        """Generate a URL for a file/document

        obj: the file object returned by Minio.list_objects()
        host: the host where the file was found
        tag: well tag number
        private: private file permissions are handled by externally. when private=True,
            an external link will be generated.
        """

        if private:
            return self.get_private_file(obj.object_name)

        return 'https://{}/{}/{}'.format(
            host,
            quote(obj.bucket_name),
            quote(obj.object_name)
        )

    def create_url_list(self, objects, host, private=False):
        """Generate a list of documents with name and url"""
        urls = list(
            map(
                lambda document: {
                    'url': self.create_url(document, host, private),

                    # split on last occurrence of '/' and return last item (supports any or no prefixes)
                    'name': document.object_name.rsplit('/', 1)[-1]
                }, objects)
        )
        return urls

    def get_prefix(self, document_id, resource='well'):
        """Helper function to determine the prefix for a given resource"""
        if resource == 'well':
            prefix = str(str('{:0<6}'.format('{:0>2}'.format(document_id // 10000))) + '/WTN ' +
                         str(document_id) + '_')
        elif resource == 'aquifer':
            prefix = str(str('{:0<5}'.format('{:0>3}'.format(document_id // 100))) + '/AQ_' +
                         str('{:0<5}'.format('{:0>5}'.format(document_id))) + '_')
        elif resource == 'driller':
            prefix = "P_%s" % document_id

        return prefix

    def format_object_name(self, object_name: str, document_id: int, resource='well'):
        """Wrapper function for getting an object name, with path and prefix, for an object and resource type"""
        return self.get_prefix(document_id, resource) + object_name

    def get_documents(self, document_id: int, resource='well', include_private=False):
        """Retrieves a list of available documents for a well or aquifer"""

        # prefix well tag numbers with a 6 digit "folder" id
        # e.g. WTA 23456 goes into prefix 020000/
        prefix = self.get_prefix(document_id, resource)

        if resource == 'well':
            public_bucket = self.public_bucket
        elif resource == 'aquifer':
            public_bucket = self.public_aquifers_bucket
        elif resource == 'driller':
            public_bucket = self.public_drillers_bucket

        objects = {}
        print(public_bucket)
        print(prefix)

        # provide all requests with a "public" collection of documents
        if self.public_client:
            pub_objects = []
            try:
                print(self.public_client.list_objects(
                        public_bucket, prefix=prefix, recursive=True))
                pub_objects = self.create_url_list(
                    self.public_client.list_objects(
                        public_bucket, prefix=prefix, recursive=True),
                    self.public_host)
                print(pub_objects)
            except:
                logger.error(
                    "Could not retrieve files from public file server")

            objects['public'] = pub_objects

        # authenticated requests also receive a "private" collection
        if include_private and not self.disable_private:
            priv_objects = []
            try:
                priv_objects = self.create_url_list(
                    self.private_client.list_objects(
                        self.private_bucket, prefix=prefix, recursive=True),
                    self.private_host, private=True)
            except:
                logger.error(
                    "Could not retrieve files from private file server", exc_info=sys.exc_info())

            objects['private'] = priv_objects

        return objects

    def get_presigned_put_url(self, object_name, bucket_name=None, private=False):
        """Retrieves the a presigned URL for putting objects into an S3 source"""
        if private:
            if bucket_name is None:
                bucket_name = self.private_bucket

            key = self.private_client.presigned_put_object(
                bucket_name, object_name, expires=timedelta(minutes=5))
        else:
            if bucket_name is None:
                bucket_name = self.public_bucket

            key = self.public_client.presigned_put_object(
                bucket_name, object_name, expires=timedelta(minutes=5))

        return key
<commit_msg>Removed superfluous logging messages from Django backend<commit_after>"""
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""
import sys
import os
import logging
from datetime import timedelta
from django.urls import reverse
from urllib.parse import quote
from minio import Minio
from gwells.settings.base import get_env_variable

logger = logging.getLogger(__name__)

class MinioClient():
    """ Load a minio client to handle public and/or private file requests

    Requires environment variables:
        S3_HOST: hostname for public document storage
        S3_ROOT_BUCKET: public Wells document storage bucket
        S3_AQUIFER_BUCKET: public Aquifers document storage buck
        MINIO_ACCESS_KEY: private storage account
        MINIO_SECRET_KEY: private storage secret
        S3_PRIVATE_HOST: private storage host (must be specified even if same as public storage)
        S3_PRIVATE_BUCKET: private storage bucket

    The optional "request" param can be set to the request that requires the minio client.
    This allows generation of full URIs including domain name.
    This is only required for generating private, local links.

    e.g.:
    def get(self, request):
        client = MinioClient(request)

    """

    def __init__(self, request=None, disable_public=False, disable_private=False):
        self.request = request

        if not disable_public:
            self.public_host = get_env_variable('S3_HOST', strict=True)
            self.public_bucket = get_env_variable(
                'S3_ROOT_BUCKET', strict=True)
            self.public_aquifers_bucket = get_env_variable('S3_AQUIFER_BUCKET', default_value="aquifer-docs")
            self.public_drillers_bucket = get_env_variable('S3_REGISTRANT_BUCKET', default_value="driller-docs")
            self.public_access_key = get_env_variable(
                'S3_PUBLIC_ACCESS_KEY', warn=False)
            self.public_secret_key = get_env_variable(
                'S3_PUBLIC_SECRET_KEY', warn=False)
            self.use_secure = int(get_env_variable(
                'S3_USE_SECURE', 1, warn=False))

            self.public_client = Minio(
                self.public_host,
                access_key=self.public_access_key,
                secret_key=self.public_secret_key,
                secure=self.use_secure
            )

        if not disable_private:
            self.private_client = self.create_private_client()

        self.disable_private = disable_private

    def create_private_client(self):
        self.private_access_key = get_env_variable('MINIO_ACCESS_KEY')
        self.private_secret_key = get_env_variable('MINIO_SECRET_KEY')
        self.private_host = get_env_variable('S3_PRIVATE_HOST')
        self.private_bucket = get_env_variable('S3_PRIVATE_BUCKET')
        self.private_aquifers_bucket = get_env_variable('S3_PRIVATE_AQUIFER_BUCKET', default_value="aquifer-docs")
        self.private_drillers_bucket = get_env_variable('S3_PRIVATE_REGISTRANT_BUCKET', default_value="driller-docs")

        return Minio(
            self.private_host,
            access_key=self.private_access_key,
            secret_key=self.private_secret_key,
            secure=self.use_secure
        )

    def get_private_file(self, object_name: str):
        """ Generates a link to a private document with name "object_name" (name includes prefixes) """
        return self.private_client.presigned_get_object(
            self.private_bucket,
            object_name,
            expires=timedelta(minutes=12))

    def create_url(self, obj, host, private=False):
        """Generate a URL for a file/document

        obj: the file object returned by Minio.list_objects()
        host: the host where the file was found
        tag: well tag number
        private: private file permissions are handled by externally. when private=True,
            an external link will be generated.
        """

        if private:
            return self.get_private_file(obj.object_name)

        return 'https://{}/{}/{}'.format(
            host,
            quote(obj.bucket_name),
            quote(obj.object_name)
        )

    def create_url_list(self, objects, host, private=False):
        """Generate a list of documents with name and url"""
        urls = list(
            map(
                lambda document: {
                    'url': self.create_url(document, host, private),

                    # split on last occurrence of '/' and return last item (supports any or no prefixes)
                    'name': document.object_name.rsplit('/', 1)[-1]
                }, objects)
        )
        return urls

    def get_prefix(self, document_id, resource='well'):
        """Helper function to determine the prefix for a given resource"""
        if resource == 'well':
            prefix = str(str('{:0<6}'.format('{:0>2}'.format(document_id // 10000))) + '/WTN ' +
                         str(document_id) + '_')
        elif resource == 'aquifer':
            prefix = str(str('{:0<5}'.format('{:0>3}'.format(document_id // 100))) + '/AQ_' +
                         str('{:0<5}'.format('{:0>5}'.format(document_id))) + '_')
        elif resource == 'driller':
            prefix = "P_%s" % document_id

        return prefix

    def format_object_name(self, object_name: str, document_id: int, resource='well'):
        """Wrapper function for getting an object name, with path and prefix, for an object and resource type"""
        return self.get_prefix(document_id, resource) + object_name

    def get_documents(self, document_id: int, resource='well', include_private=False):
        """Retrieves a list of available documents for a well or aquifer"""

        # prefix well tag numbers with a 6 digit "folder" id
        # e.g. WTA 23456 goes into prefix 020000/
        prefix = self.get_prefix(document_id, resource)

        if resource == 'well':
            public_bucket = self.public_bucket
        elif resource == 'aquifer':
            public_bucket = self.public_aquifers_bucket
        elif resource == 'driller':
            public_bucket = self.public_drillers_bucket

        objects = {}

        # provide all requests with a "public" collection of documents
        if self.public_client:
            pub_objects = []
            try:
                pub_objects = self.create_url_list(
                    self.public_client.list_objects(
                        public_bucket, prefix=prefix, recursive=True),
                    self.public_host)
            except:
                logger.error(
                    "Could not retrieve files from public file server")

            objects['public'] = pub_objects

        # authenticated requests also receive a "private" collection
        if include_private and not self.disable_private:
            priv_objects = []
            try:
                priv_objects = self.create_url_list(
                    self.private_client.list_objects(
                        self.private_bucket, prefix=prefix, recursive=True),
                    self.private_host, private=True)
            except:
                logger.error(
                    "Could not retrieve files from private file server", exc_info=sys.exc_info())

            objects['private'] = priv_objects

        return objects

    def get_presigned_put_url(self, object_name, bucket_name=None, private=False):
        """Retrieves the a presigned URL for putting objects into an S3 source"""
        if private:
            if bucket_name is None:
                bucket_name = self.private_bucket

            key = self.private_client.presigned_put_object(
                bucket_name, object_name, expires=timedelta(minutes=5))
        else:
            if bucket_name is None:
                bucket_name = self.public_bucket

            key = self.public_client.presigned_put_object(
                bucket_name, object_name, expires=timedelta(minutes=5))

        return key
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, stat, time, os, errno
from bup import metadata, options, git, index, drecurse, hlinkdb
from bup.helpers import *
from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE

class IterHelper:
    def __init__(self, l):
        self.i = iter(l)
        self.cur = None
        self.next()

    def next(self):
        try:
            self.cur = self.i.next()
        except StopIteration:
            self.cur = None
        return self.cur

def check_index(reader):
    try:
        log('check: checking forward iteration...\n')
        e = None
        d = {}
        for e in reader.forward_iter():
            if e.children_n:
                if opt.verbose:
                    log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
                                            e.name))
                assert(e.children_ofs)
                assert(e.name.endswith('/'))
                assert(not d.get(e.children_ofs))
                d[e.children_ofs] = 1
            if e.flags & index.IX_HASHVALID:
                assert(e.sha != index.EMPTY_SHA)
                assert(e.gitmode)
        assert(not e or e.name == '/')  # last entry is *always* /
        log('check: checking normal iteration...\n')
        last = None
        for e in reader:
            if last:
                assert(last > e.name)
            last = e.name
    except:
        log('index error! at %r\n' % e)
        raise
    log('check: passed.\n')

def clear_index(indexfile):
    indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
    cleared = False
    for indexfile in indexfiles:
        path = git.repo(indexfile)
        try:
            os.remove(path)
            if opt.verbose:
                log('clear: removed %s\n' % path)
            cleared = True
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise

def update_index(top, excluded_paths):
    # tmax and start must be epoch nanoseconds.
    tmax = (time.time() - 1) * 10**9
    ri = index.Reader(indexfile)
    msw = index.MetaStoreWriter(indexfile + '.meta')
    wi = index.Writer(indexfile, msw, tmax)
    rig = IterHelper(ri.iter(name=top))
    tstart = int(time.time()) * 10**9

    hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')

    hashgen = None
    if opt.fake_valid:
        def hashgen(name):
            return (GIT_MODE_FILE, index.FAKE_SHA)

    total = 0
    bup_dir = os.path.abspath(git.repo())
    for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
                                                 bup_dir=bup_dir,
                                                 excluded_paths=excluded_paths):
        if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
            sys.stdout.write('%s\n' % path)
            sys.stdout.flush()
            qprogress('Indexing: %d\r' % total)
        elif not (total % 128):
            qprogress('Indexing: %d\r' % total)
        total += 1
        while rig.cur and rig.cur.name > path:  # deleted paths
            if rig.cur.exists():
                rig.cur.set_deleted()
                rig.cur.repack()
                if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
                    hlinks.del_path(rig.cur.name)
            rig.next()
        if rig.cur and rig.cur.name == path:    # paths that already existed
            if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
                hlinks.del_path(rig.cur.name)
            if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
                hlinks.add_path(path, pst.st_dev, pst.st_ino)
            meta = metadata.from_path(path, statinfo=pst)
            # Clear these so they don't bloat the store -- they're
            # already in the index (since they vary a lot and they're
            # fixed length).  If you've noticed "tmax", you might
            # wonder why it's OK to do this, since that code may
            # adjust (mangle) the index mtime and ctime -- producing
            # fake values which must not end up in a .bupm.  However,
            # it looks like that shouldn't be possible:  (1) When
            # "save" validates the index entry, it always reads the
            # metadata from the filesytem. (2) Metadata is only
            # read/used from the index if hashvalid is true. (3) index
            # always invalidates "faked" entries, because "old != new"
            # in from_stat().
            meta.ctime = meta.mtime = meta.atime = 0
            meta_ofs = msw.store(meta)
            rig.cur.from_stat(pst, meta_ofs, tstart)
            if not (rig.cur.flags & index.IX_HASHVALID):
                if hashgen:
                    (rig.cur.gitmode, rig.cur.sha) = hashgen(path)
                    rig.cur.flags |= index.IX_HASHVALID
            if opt.fake_invalid:
                rig.cur.invalidate()
            rig.cur.repack()
            rig.next()
        else:  # new paths
            meta = metadata.from_path(path, statinfo=pst)
            # See same assignment to 0, above, for rationale.
            meta.atime = meta.mtime = meta.ctime = 0
            meta_ofs = msw.store(meta)
            wi.add(path, pst, meta_ofs, hashgen = hashgen)
            if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
                hlinks.add_path(path, pst.st_dev, pst.st_ino)

    progress('Indexing: %d, done.\n' % total)

    hlinks.prepare_save()

    if ri.exists():
        ri.save()
        wi.flush()
        if wi.count:
            wr = wi.new_reader()
            if opt.check:
                log('check: before merging: oldfile\n')
                check_index(ri)
                log('check: before merging: newfile\n')
                check_index(wr)
            mi = index.Writer(indexfile, msw, tmax)

            for e in index.merge(ri, wr):
                # FIXME: shouldn't we remove deleted entries eventually?  When?
                mi.add_ixentry(e)

            ri.close()
            mi.close()
            wr.close()
        wi.abort()
    else:
        wi.close()

    msw.close()
    hlinks.commit_save()

optspec = """
bup index <-p|m|s|u> [options...] <filenames...>
--
 Modes:
p,print    print the index entries for the given names (also works with -u)
m,modified print only added/deleted/modified files (implies -p)
s,status   print each filename with a status char (A/M/D) (implies -p)
u,update   recursively update the index entries for the given file/dir names (default if no mode is specified)
check      carefully check index file integrity
clear      clear the index
 Options:
H,hash     print the hash for each object next to its name
l,long     print more information about each file
fake-valid mark all index entries as up-to-date even if they aren't
fake-invalid mark all index entries as invalid
f,indexfile=  the name of the index file (normally BUP_DIR/bupindex)
exclude=   a path to exclude from the backup (can be used more than once)
exclude-from= a file that contains exclude paths (can be used more than once)
v,verbose  increase log output (can be used more than once)
x,xdev,one-file-system  don't cross filesystem boundaries
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

if not (opt.modified or \
        opt['print'] or \
        opt.status or \
        opt.update or \
        opt.check or \
        opt.clear):
    opt.update = 1
if (opt.fake_valid or opt.fake_invalid) and not opt.update:
    o.fatal('--fake-{in,}valid are meaningless without -u')
if opt.fake_valid and opt.fake_invalid:
    o.fatal('--fake-valid is incompatible with --fake-invalid')

# FIXME: remove this once we account for timestamp races, i.e. index;
# touch new-file; index.  It's possible for this to happen quickly
# enough that new-file ends up with the same timestamp as the first
# index, and then bup will ignore it.
tick_start = time.time()
time.sleep(1 - (tick_start - int(tick_start)))

git.check_repo_or_die()
indexfile = opt.indexfile or git.repo('bupindex')

handle_ctrl_c()

if opt.check:
    log('check: starting initial check.\n')
    check_index(index.Reader(indexfile))

if opt.clear:
    log('clear: clearing index.\n')
    clear_index(indexfile)

excluded_paths = drecurse.parse_excludes(flags)

paths = index.reduce_paths(extra)

if opt.update:
    if not extra:
        o.fatal('update mode (-u) requested but no paths given')
    for (rp,path) in paths:
        update_index(rp, excluded_paths)

if opt['print'] or opt.status or opt.modified:
    for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
        if (opt.modified
            and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
            continue
        line = ''
        if opt.status:
            if ent.is_deleted():
                line += 'D '
            elif not ent.is_valid():
                if ent.sha == index.EMPTY_SHA:
                    line += 'A '
                else:
                    line += 'M '
            else:
                line += '  '
        if opt.hash:
            line += ent.sha.encode('hex') + ' '
        if opt.long:
            line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
        print line + (name or './')

if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
    log('check: starting final check.\n')
    check_index(index.Reader(indexfile))

if saved_errors:
    log('WARNING: %d errors encountered.\n' % len(saved_errors))
    sys.exit(1)
<commit_msg>clear_index(): remove unused "cleared" variable.<commit_after>#!/usr/bin/env python

import sys, stat, time, os, errno
from bup import metadata, options, git, index, drecurse, hlinkdb
from bup.helpers import *
from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE

class IterHelper:
    def __init__(self, l):
        self.i = iter(l)
        self.cur = None
        self.next()

    def next(self):
        try:
            self.cur = self.i.next()
        except StopIteration:
            self.cur = None
        return self.cur

def check_index(reader):
    try:
        log('check: checking forward iteration...\n')
        e = None
        d = {}
        for e in reader.forward_iter():
            if e.children_n:
                if opt.verbose:
                    log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
                                            e.name))
                assert(e.children_ofs)
                assert(e.name.endswith('/'))
                assert(not d.get(e.children_ofs))
                d[e.children_ofs] = 1
            if e.flags & index.IX_HASHVALID:
                assert(e.sha != index.EMPTY_SHA)
                assert(e.gitmode)
        assert(not e or e.name == '/')  # last entry is *always* /
        log('check: checking normal iteration...\n')
        last = None
        for e in reader:
            if last:
                assert(last > e.name)
            last = e.name
    except:
        log('index error! at %r\n' % e)
        raise
    log('check: passed.\n')

def clear_index(indexfile):
    indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
    for indexfile in indexfiles:
        path = git.repo(indexfile)
        try:
            os.remove(path)
            if opt.verbose:
                log('clear: removed %s\n' % path)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise

def update_index(top, excluded_paths):
    # tmax and start must be epoch nanoseconds.
    tmax = (time.time() - 1) * 10**9
    ri = index.Reader(indexfile)
    msw = index.MetaStoreWriter(indexfile + '.meta')
    wi = index.Writer(indexfile, msw, tmax)
    rig = IterHelper(ri.iter(name=top))
    tstart = int(time.time()) * 10**9

    hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')

    hashgen = None
    if opt.fake_valid:
        def hashgen(name):
            return (GIT_MODE_FILE, index.FAKE_SHA)

    total = 0
    bup_dir = os.path.abspath(git.repo())
    for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
                                                 bup_dir=bup_dir,
                                                 excluded_paths=excluded_paths):
        if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
            sys.stdout.write('%s\n' % path)
            sys.stdout.flush()
            qprogress('Indexing: %d\r' % total)
        elif not (total % 128):
            qprogress('Indexing: %d\r' % total)
        total += 1
        while rig.cur and rig.cur.name > path:  # deleted paths
            if rig.cur.exists():
                rig.cur.set_deleted()
                rig.cur.repack()
                if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
                    hlinks.del_path(rig.cur.name)
            rig.next()
        if rig.cur and rig.cur.name == path:    # paths that already existed
            if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
                hlinks.del_path(rig.cur.name)
            if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
                hlinks.add_path(path, pst.st_dev, pst.st_ino)
            meta = metadata.from_path(path, statinfo=pst)
            # Clear these so they don't bloat the store -- they're
            # already in the index (since they vary a lot and they're
            # fixed length).  If you've noticed "tmax", you might
            # wonder why it's OK to do this, since that code may
            # adjust (mangle) the index mtime and ctime -- producing
            # fake values which must not end up in a .bupm.  However,
            # it looks like that shouldn't be possible:  (1) When
            # "save" validates the index entry, it always reads the
            # metadata from the filesytem. (2) Metadata is only
            # read/used from the index if hashvalid is true. (3) index
            # always invalidates "faked" entries, because "old != new"
            # in from_stat().
            meta.ctime = meta.mtime = meta.atime = 0
            meta_ofs = msw.store(meta)
            rig.cur.from_stat(pst, meta_ofs, tstart)
            if not (rig.cur.flags & index.IX_HASHVALID):
                if hashgen:
                    (rig.cur.gitmode, rig.cur.sha) = hashgen(path)
                    rig.cur.flags |= index.IX_HASHVALID
            if opt.fake_invalid:
                rig.cur.invalidate()
            rig.cur.repack()
            rig.next()
        else:  # new paths
            meta = metadata.from_path(path, statinfo=pst)
            # See same assignment to 0, above, for rationale.
            meta.atime = meta.mtime = meta.ctime = 0
            meta_ofs = msw.store(meta)
            wi.add(path, pst, meta_ofs, hashgen = hashgen)
            if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
                hlinks.add_path(path, pst.st_dev, pst.st_ino)

    progress('Indexing: %d, done.\n' % total)

    hlinks.prepare_save()

    if ri.exists():
        ri.save()
        wi.flush()
        if wi.count:
            wr = wi.new_reader()
            if opt.check:
                log('check: before merging: oldfile\n')
                check_index(ri)
                log('check: before merging: newfile\n')
                check_index(wr)
            mi = index.Writer(indexfile, msw, tmax)

            for e in index.merge(ri, wr):
                # FIXME: shouldn't we remove deleted entries eventually?  When?
                mi.add_ixentry(e)

            ri.close()
            mi.close()
            wr.close()
        wi.abort()
    else:
        wi.close()

    msw.close()
    hlinks.commit_save()

optspec = """
bup index <-p|m|s|u> [options...] <filenames...>
--
 Modes:
p,print    print the index entries for the given names (also works with -u)
m,modified print only added/deleted/modified files (implies -p)
s,status   print each filename with a status char (A/M/D) (implies -p)
u,update   recursively update the index entries for the given file/dir names (default if no mode is specified)
check      carefully check index file integrity
clear      clear the index
 Options:
H,hash     print the hash for each object next to its name
l,long     print more information about each file
fake-valid mark all index entries as up-to-date even if they aren't
fake-invalid mark all index entries as invalid
f,indexfile=  the name of the index file (normally BUP_DIR/bupindex)
exclude=   a path to exclude from the backup (can be used more than once)
exclude-from= a file that contains exclude paths (can be used more than once)
v,verbose  increase log output (can be used more than once)
x,xdev,one-file-system  don't cross filesystem boundaries
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

if not (opt.modified or \
        opt['print'] or \
        opt.status or \
        opt.update or \
        opt.check or \
        opt.clear):
    opt.update = 1
if (opt.fake_valid or opt.fake_invalid) and not opt.update:
    o.fatal('--fake-{in,}valid are meaningless without -u')
if opt.fake_valid and opt.fake_invalid:
    o.fatal('--fake-valid is incompatible with --fake-invalid')

# FIXME: remove this once we account for timestamp races, i.e. index;
# touch new-file; index.  It's possible for this to happen quickly
# enough that new-file ends up with the same timestamp as the first
# index, and then bup will ignore it.
tick_start = time.time()
time.sleep(1 - (tick_start - int(tick_start)))

git.check_repo_or_die()
indexfile = opt.indexfile or git.repo('bupindex')

handle_ctrl_c()

if opt.check:
    log('check: starting initial check.\n')
    check_index(index.Reader(indexfile))

if opt.clear:
    log('clear: clearing index.\n')
    clear_index(indexfile)

excluded_paths = drecurse.parse_excludes(flags)

paths = index.reduce_paths(extra)

if opt.update:
    if not extra:
        o.fatal('update mode (-u) requested but no paths given')
    for (rp,path) in paths:
        update_index(rp, excluded_paths)

if opt['print'] or opt.status or opt.modified:
    for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
        if (opt.modified
            and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
            continue
        line = ''
        if opt.status:
            if ent.is_deleted():
                line += 'D '
            elif not ent.is_valid():
                if ent.sha == index.EMPTY_SHA:
                    line += 'A '
                else:
                    line += 'M '
            else:
                line += '  '
        if opt.hash:
            line += ent.sha.encode('hex') + ' '
        if opt.long:
            line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
        print line + (name or './')

if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
    log('check: starting final check.\n')
    check_index(index.Reader(indexfile))

if saved_errors:
    log('WARNING: %d errors encountered.\n' % len(saved_errors))
    sys.exit(1)
<|endoftext|>


<commit_before>from .logging import log
from .managers import CardManager

class AuraBuff:
	def __init__(self, source, entity):
		self.source = source
		self.entity = entity
		self.tags = CardManager(self)

	def __repr__(self):
		return "<AuraBuff %r -> %r>" % (self.source, self.entity)

	def update_tags(self, tags):
		self.tags.update(tags)
		self.tick = self.source.game.tick

	def destroy(self):
		log.info("Destroying %r", self)
		self.entity.slots.remove(self)
		self.source.game.active_aura_buffs.remove(self)

	def _getattr(self, attr, i):
		value = getattr(self, attr, 0)
		if callable(value):
			return value(self.entity, i)
		return i + value

class Refresh:
	"""
	Refresh a buff or a set of tags on an entity
	"""
	def __init__(self, selector, tags=None, buff=None, priority=50):
		self.selector = selector
		self.tags = tags
		self.buff = buff
		self.priority = priority

	def trigger(self, source):
		entities = self.selector.eval(source.game, source)
		for entity in entities:
			if self.buff:
				entity.refresh_buff(source, self.buff)
			else:
				tags = {}
				for tag, value in self.tags.items():
					if not isinstance(value, int) and not callable(value):
						value = value.evaluate(source)
					tags[tag] = value

				entity.refresh_tags(source, tags)

	def __repr__(self):
		return "Refresh(%r, %r, %r)" % (self.selector, self.tags or {}, self.buff or "")

class TargetableByAuras:
	def refresh_buff(self, source, id):
		for slot in self.slots:
			if slot.source is source:
				slot.tick = source.game.tick
				break
		else:
			log.info("Aura from %r buffs %r with %r", source, self, id)
			buff = source.buff(self, id)
			buff.tick = source.game.tick
			source.game.active_aura_buffs.append(buff)

	def refresh_tags(self, source, tags):
		for slot in self.slots:
			if slot.source is source:
				slot.update_tags(tags)
				break
		else:
			buff = AuraBuff(source, self)
			log.info("Creating %r", buff)
			buff.update_tags(tags)
			self.slots.append(buff)
			source.game.active_aura_buffs.append(buff)
<commit_msg>Compare buff IDs when refreshing a buff<commit_after>from .logging import log
from .managers import CardManager

class AuraBuff:
	def __init__(self, source, entity):
		self.source = source
		self.entity = entity
		self.tags = CardManager(self)

	def __repr__(self):
		return "<AuraBuff %r -> %r>" % (self.source, self.entity)

	def update_tags(self, tags):
		self.tags.update(tags)
		self.tick = self.source.game.tick

	def destroy(self):
		log.info("Destroying %r", self)
		self.entity.slots.remove(self)
		self.source.game.active_aura_buffs.remove(self)

	def _getattr(self, attr, i):
		value = getattr(self, attr, 0)
		if callable(value):
			return value(self.entity, i)
		return i + value

class Refresh:
	"""
	Refresh a buff or a set of tags on an entity
	"""
	def __init__(self, selector, tags=None, buff=None, priority=50):
		self.selector = selector
		self.tags = tags
		self.buff = buff
		self.priority = priority

	def trigger(self, source):
		entities = self.selector.eval(source.game, source)
		for entity in entities:
			if self.buff:
				entity.refresh_buff(source, self.buff)
			else:
				tags = {}
				for tag, value in self.tags.items():
					if not isinstance(value, int) and not callable(value):
						value = value.evaluate(source)
					tags[tag] = value

				entity.refresh_tags(source, tags)

	def __repr__(self):
		return "Refresh(%r, %r, %r)" % (self.selector, self.tags or {}, self.buff or "")

class TargetableByAuras:
	def refresh_buff(self, source, id):
		for slot in self.slots:
			if slot.source is source and slot.id == id:
				slot.tick = source.game.tick
				break
		else:
			log.info("Aura from %r buffs %r with %r", source, self, id)
			buff = source.buff(self, id)
			buff.tick = source.game.tick
			source.game.active_aura_buffs.append(buff)

	def refresh_tags(self, source, tags):
		for slot in self.slots:
			if slot.source is source:
				slot.update_tags(tags)
				break
		else:
			buff = AuraBuff(source, self)
			log.info("Creating %r", buff)
			buff.update_tags(tags)
			self.slots.append(buff)
			source.game.active_aura_buffs.append(buff)
<|endoftext|>


<commit_before># This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# For more information, please refer to <http://unlicense.org/>

from exceptions import Exception
import re
import requests
from bs4 import BeautifulSoup

class RateBeer(object):
    """
    Makes getting information about beers and breweries from RateBeer.com easy.

    >>> summit_search = RateBeer().search("summit extra pale ale")

    A utility for searching RateBeer.com, finding information about beers,
    breweries, and reviews. The nature of web scraping means that this package
    is offered in perpetual beta.

    Requires BeautifulSoup4, Requests, and lxml. See
    https://github.com/alilja/ratebeer for the full README.

    """
    _BASE_URL = "http://www.ratebeer.com"

    class PageNotFound(Exception):
        """Returns the URL of the page not found."""
        pass

    class AliasedBeer(Exception):
        """Returns the old and new urls for an aliased beer."""
        def __init__(self, oldurl, newurl):
            self.oldurl = oldurl
            self.newurl = newurl

    def _get_soup(self, url):
        req = requests.get(RateBeer._BASE_URL + url, allow_redirects=True)
        if "ratebeer robot oops" in req.text.lower():
            raise RateBeer.PageNotFound(url)
        return BeautifulSoup(req.text, "lxml")

    def search(self, query):
        """Returns a list of beers and breweries that matched the search query.

        Args:
            query (string): The text of the search.

        Returns:
            A dictionary containing two lists, ``breweries`` and ``beers``.
            Each list contains a dictionary of attributes of that brewery or
            beer.
        """
        r = requests.post(RateBeer._BASE_URL + "/findbeer.asp",
                          data={"BeerName": query})
        soup = BeautifulSoup(r.text, "lxml")
        s_results = soup.find_all('table', {'class': 'results'})
        output = {"breweries": [], "beers": []}

        # Locate rows that contain the brewery and beer info
        brewers_loc = None
        beers_loc = None
        for idx, val in enumerate(soup.find_all("h1")):
            if "brewers" in val:
                brewers_loc = idx
            elif "beers" in val:
                beers_loc = idx

        # find brewery information
        if brewers_loc != None:
            s_breweries = s_results[brewers_loc-1].find_all('tr')
            for row in s_breweries:
                location = row.find('td', {'align': 'right'})
                output['breweries'].append({
                    "name": row.a.text,
                    "url": row.a.get('href'),
                    "id": re.search(r"/(?P<id>\d*)/", row.a.get('href')).group('id'),
                    "location": location.text.strip(),
                })

        # find beer information
        if beers_loc != None and not soup.find_all(text="0 beers"):
            s_beer_trs = iter(s_results[beers_loc-1].find_all('tr'))
            next(s_beer_trs)
            for row in s_beer_trs:
                link = row.find('td', 'results').a
                align_right = row.find_all("td", {'align': 'right'})
                output['beers'].append({
                    "name": link.text,
                    "url": link.get('href'),
                    "id": re.search(r"/(?P<id>\d*)/", link.get('href')).group('id'),
                    "rating": align_right[-2].text.strip(),
                    "num_ratings": align_right[-1].text,
                })
        return output

    def beer(self, url):
        """Returns information about a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"

        Returns:
            A dictionary of attributes about that beer."""
        soup = self._get_soup(url)
        output = {}

        # check for 404s
        try:
            s_contents_rows = soup.find('div', id='container').find('table').find_all('tr')
        except AttributeError:
            raise RateBeer.PageNotFound(url)
        # ratebeer pages don't actually 404, they just send you to this weird
        # "beer reference" page but the url doesn't actually change, it just
        # seems like it's all getting done server side -- so we have to look
        # for the contents h1 to see if we're looking at the beer reference or
        # not
        if "beer reference" in s_contents_rows[0].find_all('td')[1].h1.contents:
            raise RateBeer.PageNotFound(url)

        if "Also known as " in s_contents_rows[1].find_all('td')[1].div.div.contents:
            raise RateBeer.AliasedBeer(url, s_contents_rows[1].find_all('td')[1].div.div.a['href'])

        brew_info_row = s_contents_rows[1].find_all('td')[1].div.small
        brew_info = brew_info_row.text.split(u'\xa0\xa0')
        brew_info = [s.split(': ') for s in brew_info]
        keywords = {
            "RATINGS": "num_ratings",
            "MEAN": "mean",
            "WEIGHTED AVG": "weighted_avg",
            "SEASONAL": "seasonal",
            "CALORIES": "calories",
            "ABV": "abv",
            "IBU": "ibu",
        }
        for meta_name, meta_data in brew_info:
            for keyword in keywords:
                if keyword in meta_name:
                    if keyword == "MEAN":
                        meta_data = meta_data[:meta_data.find("/")]
                    if keyword == "ABV":
                        meta_data = meta_data[:-1]
                    try:
                        meta_data = float(meta_data)
                    except ValueError:
                        pass
                    output[keywords[keyword]] = meta_data
                    break

        info = s_contents_rows[1].tr.find_all('td')

        brewery_info = info[1].find('div').contents
        brewery = brewery_info[0].findAll('a')[0]
        brewed_at = None
        if 'Brewed at' in brewery_info[0].text:
            brewed_at = brewery_info[0].findAll('a')[1]

        style = brewery_info[3]
        description = s_contents_rows[1].find_all('td')[1].find(
            'div', style=(
                'border: 1px solid #e0e0e0; background: #fff; '
                'padding: 14px; color: #777;'
            )
        )

        name = s_contents_rows[0].find_all('td')[1].h1
        overall_rating = info[0].find_all('span', itemprop='average')
        style_rating = info[0].find_all('div')
        if len(style_rating) < 2:
            style_rating = None

        output['name'] = name.text.strip()
        if overall_rating:
            output['overall_rating'] = overall_rating[0].text.strip()
        if style_rating:
            output['style_rating'] = style_rating[2].div.span.text.strip()
        if brewery:
            output['brewery'] = brewery.text.strip()
            output['brewery_url'] = brewery.get('href')
        if brewed_at:
            output['brewed_at'] = brewed_at.text.strip()
            output['brewed_at_url'] = brewed_at.get('href')
        if style:
            output['style'] = style.text.strip()
        if 'No commercial description' not in description.text:
            _ = [s.extract() for s in description('small')]
            output['description'] = ' '.join([s for s in description.strings]).strip()
        return output

    def reviews(self, url, pages=1, start_page=1, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            pages (int): Number of pages to return. Must be > 0.
            start_page (int): Which page to begin results. Must be > 0.
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A list of dictionaries, containing the information about the review.
        """
        assert pages > 0, "``pages`` must be greater than 0"
        assert start_page > 0, "``start_page`` must be greater than 0"
        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if url_flag is None:
            raise ValueError("Invalid ``review_order``.")

        output = []
        for page_number in range(start_page, start_page + pages):
            complete_url = "{0}{1}/{2}/".format(url, url_flag, page_number)
            soup = self._get_soup(complete_url)
            content = soup.find('div', id='container').find('table').find_all('tr')[5]
            _ = [x.extract() for x in content.find_all('table')]  # strip ad section
            review_tuples = zip(*[iter(content.find_all('div'))] * 4)  # basically magic

            for review in review_tuples:
                detail_tuples = zip(*[iter(review[0].find_all(["big", "small"]))] * 2)
                details = dict([(
                    label.text.lower().strip().encode("ascii", "ignore"),
                    rating.text,
                ) for (label, rating) in detail_tuples])

                details.update({'text': review[3].text})
                output.append(details)
        return output

    def brewery(self, url, include_beers=True):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""
        def _find_span(search_soup, item_prop):
            output = search_soup.find('span', attrs={'itemprop': item_prop})
            output = output.text if output else None
            return output

        soup = self._get_soup(url)
        try:
            s_contents = soup.find('div', id='container').find('table').find_all('tr')[0].find_all('td')
        except AttributeError:
            raise RateBeer.PageNotFound(url)

        output = {
            'name': s_contents[8].h1.text,
            'type': re.search(r"Type: +(?P<type>[^ ]+)",
                              s_contents[8].find_all('span', 'beerfoot')[1].text).group('type'),
            'street': _find_span(s_contents[0], 'streetAddress'),
            'city': _find_span(s_contents[0], 'addressLocality'),
            'state': _find_span(s_contents[0], 'addressRegion'),
            'country': _find_span(s_contents[0], 'addressCountry'),
            'postal_code': _find_span(s_contents[0], 'postalCode'),
        }

        if include_beers:
            output.update({'beers': []})
            s_beer_trs = iter(s_contents[8].find('table', 'maintable nohover').find_all('tr'))
            next(s_beer_trs)
            for row in s_beer_trs:
                if len(row.find_all('td')) > 1:
                    beer = {
                        'name': row.a.text,
                        'url': row.a.get('href'),
                        'id': re.search(r"/(?P<id>\d*)/", row.a.get('href')).group('id'),
                        'rating': row.find_all('td')[4].text.strip(),
                        'num_ratings': row.find_all('td')[6].text.strip(),
                    }
                    output['beers'].append(beer)
        return output

    def beer_style_list(self):
        """Returns the list of beer styles from the beer styles page.

        Returns:
            A dictionary, with beer styles for keys and urls for values.
        """
        styles = {}

        soup = self._get_soup("/beerstyles/")
        columns = soup.find_all('table')[2].find_all('td')
        for column in columns:
            lines = [li for li in column.find_all('li')]
            for line in lines:
                styles[line.text] = line.a.get('href')
        return styles

    def beer_style(self, url, sort_type="overall"):
        """Get all the beers from a specific beer style page.

        Args:
            url (string): The specific url of the beer style. Looks like:
                "/beerstyles/abbey-dubbel/71/"
            sort_type (string): The sorting of the results. "overall" returns
                the highest- rated beers, while "trending" returns the newest
                and trending ones.

        Returns:
            A list of dictionaries containing the beers.
        """
        sort_type = sort_type.lower()
        url_codes = {"overall": 0, "trending": 1}
        sort_flag = url_codes.get(sort_type)
        if sort_flag is None:
            raise ValueError("Invalid ``sort_type``.")
        style_id = re.search(r"/(?P<id>\d*)/", url).group('id')

        req = requests.post(
            RateBeer._BASE_URL +
            (
                "/ajax/top-beer-by-style.asp?style={0}&sort={1}"
                "&order=0&min=10&max=9999&retired=0&new=0&mine=0&"
            )
            .format(style_id, sort_flag),
            allow_redirects=True
        )
        soup = BeautifulSoup(req.text, "lxml")
        rows = iter(soup.table.find_all('tr'))
        next(rows)

        output = []
        for row in rows:
            data = row.find_all('td')
            output.append({
                'name': data[1].text,
                'url': data[1].a.get('href'),
                'rating': data[4].text
            })
        return output
<commit_msg>Fixed incorrect overall rating scraping for some beers, e.g. /beer/michelob-bavarian-style-wheat/64779/ Rating is also returned as an int and is ignored if it is "n/a"<commit_after># This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# For more information, please refer to <http://unlicense.org/>

from exceptions import Exception
import re
import requests
from bs4 import BeautifulSoup

class RateBeer(object):
    """
    Makes getting information about beers and breweries from RateBeer.com easy.

    >>> summit_search = RateBeer().search("summit extra pale ale")

    A utility for searching RateBeer.com, finding information about beers,
    breweries, and reviews. The nature of web scraping means that this package
    is offered in perpetual beta.

    Requires BeautifulSoup4, Requests, and lxml. See
    https://github.com/alilja/ratebeer for the full README.

    """
    _BASE_URL = "http://www.ratebeer.com"

    class PageNotFound(Exception):
        """Returns the URL of the page not found."""
        pass

    class AliasedBeer(Exception):
        """Returns the old and new urls for an aliased beer."""
        def __init__(self, oldurl, newurl):
            self.oldurl = oldurl
            self.newurl = newurl

    def _get_soup(self, url):
        req = requests.get(RateBeer._BASE_URL + url, allow_redirects=True)
        if "ratebeer robot oops" in req.text.lower():
            raise RateBeer.PageNotFound(url)
        return BeautifulSoup(req.text, "lxml")

    def search(self, query):
        """Returns a list of beers and breweries that matched the search query.

        Args:
            query (string): The text of the search.

        Returns:
            A dictionary containing two lists, ``breweries`` and ``beers``.
            Each list contains a dictionary of attributes of that brewery or
            beer.
        """
        r = requests.post(RateBeer._BASE_URL + "/findbeer.asp",
                          data={"BeerName": query})
        soup = BeautifulSoup(r.text, "lxml")
        s_results = soup.find_all('table', {'class': 'results'})
        output = {"breweries": [], "beers": []}

        # Locate rows that contain the brewery and beer info
        brewers_loc = None
        beers_loc = None
        for idx, val in enumerate(soup.find_all("h1")):
            if "brewers" in val:
                brewers_loc = idx
            elif "beers" in val:
                beers_loc = idx

        # find brewery information
        if brewers_loc != None:
            s_breweries = s_results[brewers_loc-1].find_all('tr')
            for row in s_breweries:
                location = row.find('td', {'align': 'right'})
                output['breweries'].append({
                    "name": row.a.text,
                    "url": row.a.get('href'),
                    "id": re.search(r"/(?P<id>\d*)/", row.a.get('href')).group('id'),
                    "location": location.text.strip(),
                })

        # find beer information
        if beers_loc != None and not soup.find_all(text="0 beers"):
            s_beer_trs = iter(s_results[beers_loc-1].find_all('tr'))
            next(s_beer_trs)
            for row in s_beer_trs:
                link = row.find('td', 'results').a
                align_right = row.find_all("td", {'align': 'right'})
                output['beers'].append({
                    "name": link.text,
                    "url": link.get('href'),
                    "id": re.search(r"/(?P<id>\d*)/", link.get('href')).group('id'),
                    "rating": align_right[-2].text.strip(),
                    "num_ratings": align_right[-1].text,
                })
        return output

    def beer(self, url):
        """Returns information about a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"

        Returns:
            A dictionary of attributes about that beer."""
        soup = self._get_soup(url)
        output = {}

        # check for 404s
        try:
            s_contents_rows = soup.find('div', id='container').find('table').find_all('tr')
        except AttributeError:
            raise RateBeer.PageNotFound(url)
        # ratebeer pages don't actually 404, they just send you to this weird
        # "beer reference" page but the url doesn't actually change, it just
        # seems like it's all getting done server side -- so we have to look
        # for the contents h1 to see if we're looking at the beer reference or
        # not
        if "beer reference" in s_contents_rows[0].find_all('td')[1].h1.contents:
            raise RateBeer.PageNotFound(url)

        if "Also known as " in s_contents_rows[1].find_all('td')[1].div.div.contents:
            raise RateBeer.AliasedBeer(url, s_contents_rows[1].find_all('td')[1].div.div.a['href'])

        brew_info_row = s_contents_rows[1].find_all('td')[1].div.small
        brew_info = brew_info_row.text.split(u'\xa0\xa0')
        brew_info = [s.split(': ') for s in brew_info]
        keywords = {
            "RATINGS": "num_ratings",
            "MEAN": "mean",
            "WEIGHTED AVG": "weighted_avg",
            "SEASONAL": "seasonal",
            "CALORIES": "calories",
            "ABV": "abv",
            "IBU": "ibu",
        }
        for meta_name, meta_data in brew_info:
            for keyword in keywords:
                if keyword in meta_name:
                    if keyword == "MEAN":
                        meta_data = meta_data[:meta_data.find("/")]
                    if keyword == "ABV":
                        meta_data = meta_data[:-1]
                    try:
                        meta_data = float(meta_data)
                    except ValueError:
                        pass
                    output[keywords[keyword]] = meta_data
                    break

        info = s_contents_rows[1].tr.find_all('td')

        brewery_info = info[1].find('div').contents
        brewery = brewery_info[0].findAll('a')[0]
        brewed_at = None
        if 'Brewed at' in brewery_info[0].text:
            brewed_at = brewery_info[0].findAll('a')[1]

        style = brewery_info[3]
        description = s_contents_rows[1].find_all('td')[1].find(
            'div', style=(
                'border: 1px solid #e0e0e0; background: #fff; '
                'padding: 14px; color: #777;'
            )
        )

        name = s_contents_rows[0].find_all('td')[1].h1
        ratings = info[0].findAll('div')
        if len(ratings) > 1:
            overall_rating = ratings[1].findAll('span')
            style_rating = ratings[3].findAll('span')
        else:
            overall_rating = None
            style_rating = None

        output['name'] = name.text.strip()
        if overall_rating and overall_rating[1].text != 'n/a':
            output['overall_rating'] = int(overall_rating[1].text)
        if style_rating and style_rating[0].text != 'n/a':
            output['style_rating'] = int(style_rating[0].text)
        if brewery:
            output['brewery'] = brewery.text.strip()
            output['brewery_url'] = brewery.get('href')
        if brewed_at:
            output['brewed_at'] = brewed_at.text.strip()
            output['brewed_at_url'] = brewed_at.get('href')
        if style:
            output['style'] = style.text.strip()
        if 'No commercial description' not in description.text:
            _ = [s.extract() for s in description('small')]
            output['description'] = ' '.join([s for s in description.strings]).strip()
        return output

    def reviews(self, url, pages=1, start_page=1, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            pages (int): Number of pages to return. Must be > 0.
            start_page (int): Which page to begin results. Must be > 0.
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A list of dictionaries, containing the information about the review.
        """
        assert pages > 0, "``pages`` must be greater than 0"
        assert start_page > 0, "``start_page`` must be greater than 0"
        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if url_flag is None:
            raise ValueError("Invalid ``review_order``.")

        output = []
        for page_number in range(start_page, start_page + pages):
            complete_url = "{0}{1}/{2}/".format(url, url_flag, page_number)
            soup = self._get_soup(complete_url)
            content = soup.find('div', id='container').find('table').find_all('tr')[5]
            _ = [x.extract() for x in content.find_all('table')]  # strip ad section
            review_tuples = zip(*[iter(content.find_all('div'))] * 4)  # basically magic

            for review in review_tuples:
                detail_tuples = zip(*[iter(review[0].find_all(["big", "small"]))] * 2)
                details = dict([(
                    label.text.lower().strip().encode("ascii", "ignore"),
                    rating.text,
                ) for (label, rating) in detail_tuples])

                details.update({'text': review[3].text})
                output.append(details)
        return output

    def brewery(self, url, include_beers=True):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""
        def _find_span(search_soup, item_prop):
            output = search_soup.find('span', attrs={'itemprop': item_prop})
            output = output.text if output else None
            return output

        soup = self._get_soup(url)
        try:
            s_contents = soup.find('div', id='container').find('table').find_all('tr')[0].find_all('td')
        except AttributeError:
            raise RateBeer.PageNotFound(url)

        output = {
            'name': s_contents[8].h1.text,
            'type': re.search(r"Type: +(?P<type>[^ ]+)",
                              s_contents[8].find_all('span', 'beerfoot')[1].text).group('type'),
            'street': _find_span(s_contents[0], 'streetAddress'),
            'city': _find_span(s_contents[0], 'addressLocality'),
            'state': _find_span(s_contents[0], 'addressRegion'),
            'country': _find_span(s_contents[0], 'addressCountry'),
            'postal_code': _find_span(s_contents[0], 'postalCode'),
        }

        if include_beers:
            output.update({'beers': []})
            s_beer_trs = iter(s_contents[8].find('table', 'maintable nohover').find_all('tr'))
            next(s_beer_trs)
            for row in s_beer_trs:
                if len(row.find_all('td')) > 1:
                    beer = {
                        'name': row.a.text,
                        'url': row.a.get('href'),
                        'id': re.search(r"/(?P<id>\d*)/", row.a.get('href')).group('id'),
                        'rating': row.find_all('td')[4].text.strip(),
                        'num_ratings': row.find_all('td')[6].text.strip(),
                    }
                    output['beers'].append(beer)
        return output

    def beer_style_list(self):
        """Returns the list of beer styles from the beer styles page.

        Returns:
            A dictionary, with beer styles for keys and urls for values.
        """
        styles = {}

        soup = self._get_soup("/beerstyles/")
        columns = soup.find_all('table')[2].find_all('td')
        for column in columns:
            lines = [li for li in column.find_all('li')]
            for line in lines:
                styles[line.text] = line.a.get('href')
        return styles

    def beer_style(self, url, sort_type="overall"):
        """Get all the beers from a specific beer style page.

        Args:
            url (string): The specific url of the beer style. Looks like:
                "/beerstyles/abbey-dubbel/71/"
            sort_type (string): The sorting of the results. "overall" returns
                the highest- rated beers, while "trending" returns the newest
                and trending ones.

        Returns:
            A list of dictionaries containing the beers.
        """
        sort_type = sort_type.lower()
        url_codes = {"overall": 0, "trending": 1}
        sort_flag = url_codes.get(sort_type)
        if sort_flag is None:
            raise ValueError("Invalid ``sort_type``.")
        style_id = re.search(r"/(?P<id>\d*)/", url).group('id')

        req = requests.post(
            RateBeer._BASE_URL +
            (
                "/ajax/top-beer-by-style.asp?style={0}&sort={1}"
                "&order=0&min=10&max=9999&retired=0&new=0&mine=0&"
            )
            .format(style_id, sort_flag),
            allow_redirects=True
        )
        soup = BeautifulSoup(req.text, "lxml")
        rows = iter(soup.table.find_all('tr'))
        next(rows)

        output = []
        for row in rows:
            data = row.find_all('td')
            output.append({
                'name': data[1].text,
                'url': data[1].a.get('href'),
                'rating': data[4].text
            })
        return output
<|endoftext|>