<commit_before>"""
DMI SPLITTER-UPPER THING

Makes merging sprites a hell of a lot easier.
by N3X15 <nexis@7chan.org>

Requires PIL and apng.py (included)
Written for Python 2.7.
"""

import sys, os, traceback, fnmatch, argparse

from com.byond.DMI import DMI

args = ()

def main():
	opt = argparse.ArgumentParser()  # version='0.1')
	opt.add_argument('-p', '--suppress-post-processing', dest='suppress_post_process', default=False, action='store_true')
	command = opt.add_subparsers(help='The command you wish to execute', dest='MODE')

	_disassemble = command.add_parser('disassemble', help='Disassemble a single DMI file to a destination directory')
	_disassemble.add_argument('file', type=str, help='The DMI file to disassemble.', metavar='file.dmi')
	_disassemble.add_argument('destination', type=str, help='The directory in which to dump the resulting images.', metavar='dest/')

	_disassemble_all = command.add_parser('disassemble-all', help='Disassemble a directory of DMI files to a destination directory')
	_disassemble_all.add_argument('source', type=str, help='The DMI files to disassemble.', metavar='source/')
	_disassemble_all.add_argument('destination', type=str, help='The directory in which to dump the resulting images.', metavar='dest/')

	_compile = command.add_parser('compile', help='Compile a .dmi.mak file')
	_compile.add_argument('makefile', type=str, help='The .dmi.mak file to compile.', metavar='file.dmi.mak')
	_compile.add_argument('destination', type=str, help='The location of the resulting .dmi file.', metavar='file.dmi')

	_compare = command.add_parser('compare', help='Compare two DMI files and note the differences')
	_compare.add_argument('theirs', type=str, help='One side of the difference', metavar='theirs.dmi')
	_compare.add_argument('mine', type=str, help='The other side.', metavar='mine.dmi')

	_compare_all = command.add_parser('compare-all', help='Compare two DMI file directories and note the differences')
	_compare_all.add_argument('theirs', type=str, help='One side of the difference', metavar='theirs/')
	_compare_all.add_argument('mine', type=str, help='The other side.', metavar='mine/')
	_compare_all.add_argument('report', type=str, help='The file the report is saved to', metavar='report.txt')

	_get_dmi_data = command.add_parser('get-dmi-data', help='Extract DMI header')
	_get_dmi_data.add_argument('file', type=str, help='DMI file', metavar='file.dmi')
	_get_dmi_data.add_argument('dest', type=str, help='The file where the DMI header will be saved', metavar='dest.txt')

	_set_dmi_data = command.add_parser('set-dmi-data', help='Set DMI header')
	_set_dmi_data.add_argument('file', type=str, help='One side of the difference', metavar='file.dmi')
	_set_dmi_data.add_argument('metadata', type=str, help='DMI header file', metavar='metadata.txt')

	args = opt.parse_args()
	print(args)
	if args.MODE == 'compile':
		make_dmi(args.makefile, args.destination, args)
	if args.MODE == 'compare':
		compare(args.theirs, args.mine, args, sys.stdout)
	if args.MODE == 'compare-all':
		compare_all(args.theirs, args.mine, args.report, args)
	elif args.MODE == 'disassemble':
		disassemble(args.file, args.destination, args)
	elif args.MODE == 'disassemble-all':
		disassemble_all(args.source, args.destination, args)
	elif args.MODE == 'get-dmi-data':
		get_dmi_data(args.file, args.dest, args)
	elif args.MODE == 'set-dmi-data':
		set_dmi_data(args.file, args.metadata, args)
	else:
		print('!!! Error, unknown MODE=%r' % args.MODE)

class ModeAction(argparse.Action):
	def __call__(self, parser, namespace, values, option_string=None):
		# print('%s %s %s' % (namespace, values, option_string))
		namespace.MODE = self.dest
		namespace.args = values

def get_dmi_data(path, dest, parser):
	if(os.path.isfile(path)):
		dmi = DMI(path)
		with open(dest,'w') as f:
			f.write(dmi.getHeader())

def set_dmi_data(path, headerFile, parser):
	if(os.path.isfile(path)):
		dmi = DMI(path)
		with open(headerFile, 'r') as f:
			dmi.setHeader(f.read(), path)

def make_dmi(path, dest, parser):
	if(os.path.isfile(path)):
		dmi = None
		try:
			dmi = DMI(dest)
			dmi.make(path)
			dmi.save(dest)
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (dmi.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(dmi.states))
			print('Image h/w: %s' % repr(dmi.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())

def disassemble(path, to, parser):
	print('\tD %s -> %s' % (path, to))
	if(os.path.isfile(path)):
		dmi = None
		try:
			dmi = DMI(path)
			dmi.extractTo(to, parser.suppress_post_process)
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (dmi.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(dmi.states))
			print('Image h/w: %s' % repr(dmi.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())

def compare(theirsfile, minefile, parser, reportstream):
	#print('\tD %s -> %s' % (theirsfile, minefile))
	theirs = []
	mine = []
	states = []
	o = ''
	if(os.path.isfile(theirsfile)):
		try:
			theirsDMI = DMI(theirsfile)
			theirsDMI.loadMetadata()
			theirs = theirsDMI.states
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (theirs.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(theirs.states))
			print('Image h/w: %s' % repr(theirs.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())
			o += "\n {0}: Received error, continuing: {1}".format(theirsfile,traceback.format_exc())
		for stateName in theirs:
			if stateName not in states:
				states.append(stateName)
	if(os.path.isfile(minefile)):
		try:
			mineDMI = DMI(minefile)
			mineDMI.loadMetadata()
			mine = mineDMI.states
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (mine.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(mine.states))
			print('Image h/w: %s' % repr(mine.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())
			o += "\n {0}: Received error, continuing: {1}".format(minefile,traceback.format_exc())
		for stateName in mine:
			if stateName not in states:
				states.append(stateName)
	for state in sorted(states):
		inTheirs = state in theirs
		inMine = state in mine
		if inTheirs and not inMine:
			o += '\n + {1}'.format(minefile, state)
		elif not inTheirs and inMine:
			o += '\n - {1}'.format(theirsfile, state)
		elif inTheirs and inMine:
			if theirs[state].ToString() != mine[state].ToString():
				o += '\n - {0}: {1}'.format(state,mine[state].ToString())
				o += '\n + {0}: {1}'.format(state,theirs[state].ToString())
	if o != '':
		reportstream.write('\n--- {0}'.format(theirsfile))
		reportstream.write('\n+++ {0}'.format(minefile))
		reportstream.write(o)

def disassemble_all(in_dir, out_dir, parser):
	print('D_A %s -> %s' % (in_dir, out_dir))
	for root, dirnames, filenames in os.walk(in_dir):
		for filename in fnmatch.filter(filenames, '*.dmi'):
			path = os.path.join(root, filename)
			to = os.path.join(out_dir, path.replace(in_dir, '').replace(os.path.basename(path), ''))
			disassemble(path, to, parser)

def compare_all(in_dir, out_dir, report, parser):
	with open(report, 'w') as report:
		report.write('# DMITool Difference Report: {0} {1}'.format(os.path.abspath(in_dir), os.path.abspath(out_dir)))
		for root, dirnames, filenames in os.walk(in_dir):
			for filename in fnmatch.filter(filenames, '*.dmi'):
				path = os.path.join(root, filename)
				to = os.path.join(out_dir, path.replace(in_dir, '').replace(os.path.basename(path), ''))
				to = os.path.join(to, filename)
				path = os.path.abspath(path)
				to = os.path.abspath(to)
				compare(path, to, parser, report)

if __name__ == '__main__':
	main()
<commit_msg>Add .new.dmi support to the compare tools.<commit_after>"""
DMI SPLITTER-UPPER THING

Makes merging sprites a hell of a lot easier.
by N3X15 <nexis@7chan.org>

Requires PIL and apng.py (included)
Written for Python 2.7.
"""

import sys, os, traceback, fnmatch, argparse

from com.byond.DMI import DMI

args = ()

def main():
	opt = argparse.ArgumentParser()  # version='0.1')
	opt.add_argument('-p', '--suppress-post-processing', dest='suppress_post_process', default=False, action='store_true')
	command = opt.add_subparsers(help='The command you wish to execute', dest='MODE')

	_disassemble = command.add_parser('disassemble', help='Disassemble a single DMI file to a destination directory')
	_disassemble.add_argument('file', type=str, help='The DMI file to disassemble.', metavar='file.dmi')
	_disassemble.add_argument('destination', type=str, help='The directory in which to dump the resulting images.', metavar='dest/')

	_disassemble_all = command.add_parser('disassemble-all', help='Disassemble a directory of DMI files to a destination directory')
	_disassemble_all.add_argument('source', type=str, help='The DMI files to disassemble.', metavar='source/')
	_disassemble_all.add_argument('destination', type=str, help='The directory in which to dump the resulting images.', metavar='dest/')

	_compile = command.add_parser('compile', help='Compile a .dmi.mak file')
	_compile.add_argument('makefile', type=str, help='The .dmi.mak file to compile.', metavar='file.dmi.mak')
	_compile.add_argument('destination', type=str, help='The location of the resulting .dmi file.', metavar='file.dmi')

	_compare = command.add_parser('compare', help='Compare two DMI files and note the differences')
	_compare.add_argument('theirs', type=str, help='One side of the difference', metavar='theirs.dmi')
	_compare.add_argument('mine', type=str, help='The other side.', metavar='mine.dmi')

	_compare_all = command.add_parser('compare-all', help='Compare two DMI file directories and note the differences')
	_compare_all.add_argument('theirs', type=str, help='One side of the difference', metavar='theirs/')
	_compare_all.add_argument('mine', type=str, help='The other side.', metavar='mine/')
	_compare_all.add_argument('report', type=str, help='The file the report is saved to', metavar='report.txt')

	_get_dmi_data = command.add_parser('get-dmi-data', help='Extract DMI header')
	_get_dmi_data.add_argument('file', type=str, help='DMI file', metavar='file.dmi')
	_get_dmi_data.add_argument('dest', type=str, help='The file where the DMI header will be saved', metavar='dest.txt')

	_set_dmi_data = command.add_parser('set-dmi-data', help='Set DMI header')
	_set_dmi_data.add_argument('file', type=str, help='One side of the difference', metavar='file.dmi')
	_set_dmi_data.add_argument('metadata', type=str, help='DMI header file', metavar='metadata.txt')

	args = opt.parse_args()
	print(args)
	if args.MODE == 'compile':
		make_dmi(args.makefile, args.destination, args)
	if args.MODE == 'compare':
		compare(args.theirs, args.mine, args, sys.stdout)
	if args.MODE == 'compare-all':
		compare_all(args.theirs, args.mine, args.report, args)
	elif args.MODE == 'disassemble':
		disassemble(args.file, args.destination, args)
	elif args.MODE == 'disassemble-all':
		disassemble_all(args.source, args.destination, args)
	elif args.MODE == 'get-dmi-data':
		get_dmi_data(args.file, args.dest, args)
	elif args.MODE == 'set-dmi-data':
		set_dmi_data(args.file, args.metadata, args)
	else:
		print('!!! Error, unknown MODE=%r' % args.MODE)

class ModeAction(argparse.Action):
	def __call__(self, parser, namespace, values, option_string=None):
		# print('%s %s %s' % (namespace, values, option_string))
		namespace.MODE = self.dest
		namespace.args = values

def get_dmi_data(path, dest, parser):
	if(os.path.isfile(path)):
		dmi = DMI(path)
		with open(dest,'w') as f:
			f.write(dmi.getHeader())

def set_dmi_data(path, headerFile, parser):
	if(os.path.isfile(path)):
		dmi = DMI(path)
		with open(headerFile, 'r') as f:
			dmi.setHeader(f.read(), path)

def make_dmi(path, dest, parser):
	if(os.path.isfile(path)):
		dmi = None
		try:
			dmi = DMI(dest)
			dmi.make(path)
			dmi.save(dest)
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (dmi.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(dmi.states))
			print('Image h/w: %s' % repr(dmi.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())

def disassemble(path, to, parser):
	print('\tD %s -> %s' % (path, to))
	if(os.path.isfile(path)):
		dmi = None
		try:
			dmi = DMI(path)
			dmi.extractTo(to, parser.suppress_post_process)
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (dmi.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(dmi.states))
			print('Image h/w: %s' % repr(dmi.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())

def compare(theirsfile, minefile, parser, reportstream):
	#print('\tD %s -> %s' % (theirsfile, minefile))
	theirs = []
	theirsDMI=None
	mine = []
	mineDMI=None
	states = []
	new2mine = DMI()

	o = ''
	if(os.path.isfile(theirsfile)):
		try:
			theirsDMI = DMI(theirsfile)
			theirsDMI.loadMetadata()
			theirs = theirsDMI.states
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (theirs.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(theirs.states))
			print('Image h/w: %s' % repr(theirs.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())
			o += "\n {0}: Received error, continuing: {1}".format(theirsfile,traceback.format_exc())
		for stateName in theirs:
			if stateName not in states:
				states.append(stateName)
	if(os.path.isfile(minefile)):
		try:
			mineDMI = DMI(minefile)
			mineDMI.loadMetadata()
			mine = mineDMI.states
		except SystemError as e:
			print("!!! Received SystemError in %s, halting: %s" % (mine.filename, traceback.format_exc(e)))
			print('# of cells: %d' % len(mine.states))
			print('Image h/w: %s' % repr(mine.size))
			sys.exit(1)
		except Exception as e:
			print("Received error, continuing: %s" % traceback.format_exc())
			o += "\n {0}: Received error, continuing: {1}".format(minefile,traceback.format_exc())
		for stateName in mine:
			if stateName not in states:
				states.append(stateName)
	for state in sorted(states):
		inTheirs = state in theirs
		inMine = state in mine
		if inTheirs and not inMine:
			o += '\n + {1}'.format(minefile, state)
			new2mine.states[state]=theirsDMI.states[state]
		elif not inTheirs and inMine:
			o += '\n - {1}'.format(theirsfile, state)
		elif inTheirs and inMine:
			if theirs[state].ToString() != mine[state].ToString():
				o += '\n - {0}: {1}'.format(state,mine[state].ToString())
				o += '\n + {0}: {1}'.format(state,theirs[state].ToString())
	if o != '':
		reportstream.write('\n--- {0}'.format(theirsfile))
		reportstream.write('\n+++ {0}'.format(minefile))
		reportstream.write(o)
		if len(new2mine.states)>0:
			new2mine.save(minefile.replace('.dmi','.new.dmi'))

def disassemble_all(in_dir, out_dir, parser):
	print('D_A %s -> %s' % (in_dir, out_dir))
	for root, dirnames, filenames in os.walk(in_dir):
		for filename in fnmatch.filter(filenames, '*.dmi'):
			path = os.path.join(root, filename)
			to = os.path.join(out_dir, path.replace(in_dir, '').replace(os.path.basename(path), ''))
			disassemble(path, to, parser)

def compare_all(in_dir, out_dir, report, parser):
	with open(report, 'w') as report:
		report.write('# DMITool Difference Report: {0} {1}'.format(os.path.abspath(in_dir), os.path.abspath(out_dir)))
		for root, dirnames, filenames in os.walk(in_dir):
			for filename in fnmatch.filter(filenames, '*.dmi'):
				path = os.path.join(root, filename)
				to = os.path.join(out_dir, path.replace(in_dir, '').replace(os.path.basename(path), ''))
				to = os.path.join(to, filename)
				path = os.path.abspath(path)
				to = os.path.abspath(to)
				compare(path, to, parser, report)

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>'''
This module hooks fast.ai Learners to Weights & Biases through a callback.
Requested logged data can be configured through the callback constructor.

Examples:
    WandbCallback can be used when initializing the Learner::

    ```
        from wandb.fastai import WandbCallback
        [...]
        learn = Learner(data, ..., callback_fns=WandbCallback)
        learn.fit(epochs)
    ```

    Custom parameters can be given using functools.partial::

    ```
        from wandb.fastai import WandbCallback
        from functools import partial
        [...]
        learn = Learner(data, ..., callback_fns=partial(WandbCallback, ...))
        learn.fit(epochs)
    ```

    Finally, it is possible to use WandbCallback only when starting
    training. In this case it must be instantiated::

    ```
        learn.fit(..., callbacks=WandbCallback(learn))
    ```

    or, with custom parameters::

    ```
        learn.fit(..., callbacks=WandbCallback(learn, ...))
    ```
'''
import wandb
import fastai
from fastai.callbacks import TrackerCallback
from pathlib import Path
import random
try:
    import matplotlib
    matplotlib.use('Agg')  # non-interactive backend (avoid tkinter issues)
    import matplotlib.pyplot as plt
except:
    print('Warning: matplotlib required if logging sample image predictions')

class WandbCallback(TrackerCallback):
    """
    Automatically saves model topology, losses & metrics.
    Optionally logs weights, gradients, sample predictions and best trained model.

    Args:
        learn (fastai.basic_train.Learner): the fast.ai learner to hook.
        log (str): "gradients", "parameters", "all", or None. Losses & metrics are always logged.
        save_model (bool): save model at the end of each epoch. It will also load best model at the end of training.
        monitor (str): metric to monitor for saving best model. None uses default TrackerCallback monitor value.
        mode (str): "auto", "min" or "max" to compare "monitor" values and define best model.
        input_type (str): "images" or None. Used to display sample predictions.
        validation_data (list): data used for sample predictions if input_type is set.
        predictions (int): number of predictions to make if input_type is set and validation_data is None.
        seed (int): initialize random generator for sample predictions if input_type is set and validation_data is None.
    """

    # Record if watch has been called previously (even in another instance)
    _watch_called = False

    def __init__(self,
                 learn,
                 log="gradients",
                 save_model=True,
                 monitor=None,
                 mode='auto',
                 input_type=None,
                 validation_data=None,
                 predictions=36,
                 seed=12345):

        # Check if wandb.init has been called
        if wandb.run is None:
            raise ValueError(
                'You must call wandb.init() before WandbCallback()')

        # Adapted from fast.ai "SaveModelCallback"
        if monitor is None:
            # use default TrackerCallback monitor value
            super().__init__(learn, mode=mode)
        else:
            super().__init__(learn, monitor=monitor, mode=mode)
        self.save_model = save_model
        self.model_path = Path(wandb.run.dir) / 'bestmodel.pth'

        self.log = log
        self.input_type = input_type
        self.best = None

        # Select items for sample predictions to see evolution along training
        self.validation_data = validation_data
        if input_type and not self.validation_data:
            wandbRandom = random.Random(seed)  # For repeatability
            predictions = min(predictions, len(learn.data.valid_ds))
            indices = wandbRandom.sample(range(len(learn.data.valid_ds)),
                                         predictions)
            self.validation_data = [learn.data.valid_ds[i] for i in indices]

    def on_train_begin(self, **kwargs):
        "Call watch method to log model topology, gradients & weights"

        # Set self.best, method inherited from "TrackerCallback" by "SaveModelCallback"
        super().on_train_begin()

        # Ensure we don't call "watch" multiple times
        if not WandbCallback._watch_called:
            WandbCallback._watch_called = True

            # Logs model topology and optionally gradients and weights
            wandb.watch(self.learn.model, log=self.log)

    def on_epoch_end(self, epoch, smooth_loss, last_metrics, **kwargs):
        "Logs training loss, validation loss and custom metrics & log prediction samples & save model"

        if self.save_model:
            # Adapted from fast.ai "SaveModelCallback"
            current = self.get_monitor_value()
            if current is not None and self.operator(current, self.best):
                print(
                    'Better model found at epoch {} with {} value: {}.'.format(
                        epoch, self.monitor, current))
                self.best = current

                # Save within wandb folder
                with self.model_path.open('wb') as model_file:
                    self.learn.save(model_file)

        # Log sample predictions if learn.predict is available
        if self.validation_data:
            try:
                self._wandb_log_predictions()
            except FastaiError as e:
                wandb.termwarn(e.message)
                self.validation_data = None  # prevent from trying again on next loop
            except Exception as e:
                wandb.termwarn("Unable to log prediction samples.\n{}".format(e))
                self.validation_data=None  # prevent from trying again on next loop

        # Log losses & metrics
        # Adapted from fast.ai "CSVLogger"
        logs = {
            name: stat
            for name, stat in list(
                zip(self.learn.recorder.names, [epoch, smooth_loss] +
                    last_metrics))
        }
        wandb.log(logs)

    def on_train_end(self, **kwargs):
        "Load the best model."

        if self.save_model:
            # Adapted from fast.ai "SaveModelCallback"
            if self.model_path.is_file():
                with self.model_path.open('rb') as model_file:
                    self.learn.load(model_file, purge=False)
                    print('Loaded best saved model from {}'.format(
                        self.model_path))

    def _wandb_log_predictions(self):
        "Log prediction samples"

        pred_log = []

        for x, y in self.validation_data:
            try:
                pred=self.learn.predict(x)
            except:
                raise FastaiError('Unable to run "predict" method from Learner to log prediction samples.')

            # scalar -> likely to be a category
            if not pred[1].shape:
                pred_log.append(
                    wandb.Image(
                        x.data,
                        caption='Ground Truth: {}\nPrediction: {}'.format(
                            y, pred[0])))

            # most vision datasets have a "show" function we can use
            elif hasattr(x, "show"):
                # log input data
                pred_log.append(
                    wandb.Image(x.data, caption='Input data', grouping=3))

                # log label and prediction
                for im, capt in ((pred[0], "Prediction"),
                                 (y, "Ground Truth")):
                    # Resize plot to image resolution
                    # from https://stackoverflow.com/a/13714915
                    my_dpi = 100
                    fig = plt.figure(frameon=False, dpi=my_dpi)
                    h, w = x.size
                    fig.set_size_inches(w / my_dpi, h / my_dpi)
                    ax = plt.Axes(fig, [0., 0., 1., 1.])
                    ax.set_axis_off()
                    fig.add_axes(ax)

                    # Superpose label or prediction to input image
                    x.show(ax=ax, y=im)
                    pred_log.append(wandb.Image(fig, caption=capt))
                    plt.close(fig)

            # likely to be an image
            elif hasattr(y, "shape") and (
                (len(y.shape) == 2) or
                    (len(y.shape) == 3 and y.shape[0] in [1, 3, 4])):

                pred_log.extend([
                    wandb.Image(x.data, caption='Input data', grouping=3),
                    wandb.Image(pred[0].data, caption='Prediction'),
                    wandb.Image(y.data, caption='Ground Truth')
                ])

            # we just log input data
            else:
                pred_log.append(wandb.Image(x.data, caption='Input data'))

            wandb.log({"Prediction Samples": pred_log}, commit=False)

class FastaiError(wandb.Error):
    pass
<commit_msg>fix(fastai): handle MultiCategoryList labels (#963)<commit_after>'''
This module hooks fast.ai Learners to Weights & Biases through a callback.
Requested logged data can be configured through the callback constructor.

Examples:
    WandbCallback can be used when initializing the Learner::

    ```
        from wandb.fastai import WandbCallback
        [...]
        learn = Learner(data, ..., callback_fns=WandbCallback)
        learn.fit(epochs)
    ```

    Custom parameters can be given using functools.partial::

    ```
        from wandb.fastai import WandbCallback
        from functools import partial
        [...]
        learn = Learner(data, ..., callback_fns=partial(WandbCallback, ...))
        learn.fit(epochs)
    ```

    Finally, it is possible to use WandbCallback only when starting
    training. In this case it must be instantiated::

    ```
        learn.fit(..., callbacks=WandbCallback(learn))
    ```

    or, with custom parameters::

    ```
        learn.fit(..., callbacks=WandbCallback(learn, ...))
    ```
'''
import wandb
import fastai
from fastai.callbacks import TrackerCallback
from pathlib import Path
import random
try:
    import matplotlib
    matplotlib.use('Agg')  # non-interactive backend (avoid tkinter issues)
    import matplotlib.pyplot as plt
except:
    print('Warning: matplotlib required if logging sample image predictions')

class WandbCallback(TrackerCallback):
    """
    Automatically saves model topology, losses & metrics.
    Optionally logs weights, gradients, sample predictions and best trained model.

    Args:
        learn (fastai.basic_train.Learner): the fast.ai learner to hook.
        log (str): "gradients", "parameters", "all", or None. Losses & metrics are always logged.
        save_model (bool): save model at the end of each epoch. It will also load best model at the end of training.
        monitor (str): metric to monitor for saving best model. None uses default TrackerCallback monitor value.
        mode (str): "auto", "min" or "max" to compare "monitor" values and define best model.
        input_type (str): "images" or None. Used to display sample predictions.
        validation_data (list): data used for sample predictions if input_type is set.
        predictions (int): number of predictions to make if input_type is set and validation_data is None.
        seed (int): initialize random generator for sample predictions if input_type is set and validation_data is None.
    """

    # Record if watch has been called previously (even in another instance)
    _watch_called = False

    def __init__(self,
                 learn,
                 log="gradients",
                 save_model=True,
                 monitor=None,
                 mode='auto',
                 input_type=None,
                 validation_data=None,
                 predictions=36,
                 seed=12345):

        # Check if wandb.init has been called
        if wandb.run is None:
            raise ValueError(
                'You must call wandb.init() before WandbCallback()')

        # Adapted from fast.ai "SaveModelCallback"
        if monitor is None:
            # use default TrackerCallback monitor value
            super().__init__(learn, mode=mode)
        else:
            super().__init__(learn, monitor=monitor, mode=mode)
        self.save_model = save_model
        self.model_path = Path(wandb.run.dir) / 'bestmodel.pth'

        self.log = log
        self.input_type = input_type
        self.best = None

        # Select items for sample predictions to see evolution along training
        self.validation_data = validation_data
        if input_type and not self.validation_data:
            wandbRandom = random.Random(seed)  # For repeatability
            predictions = min(predictions, len(learn.data.valid_ds))
            indices = wandbRandom.sample(range(len(learn.data.valid_ds)),
                                         predictions)
            self.validation_data = [learn.data.valid_ds[i] for i in indices]

    def on_train_begin(self, **kwargs):
        "Call watch method to log model topology, gradients & weights"

        # Set self.best, method inherited from "TrackerCallback" by "SaveModelCallback"
        super().on_train_begin()

        # Ensure we don't call "watch" multiple times
        if not WandbCallback._watch_called:
            WandbCallback._watch_called = True

            # Logs model topology and optionally gradients and weights
            wandb.watch(self.learn.model, log=self.log)

    def on_epoch_end(self, epoch, smooth_loss, last_metrics, **kwargs):
        "Logs training loss, validation loss and custom metrics & log prediction samples & save model"

        if self.save_model:
            # Adapted from fast.ai "SaveModelCallback"
            current = self.get_monitor_value()
            if current is not None and self.operator(current, self.best):
                print(
                    'Better model found at epoch {} with {} value: {}.'.format(
                        epoch, self.monitor, current))
                self.best = current

                # Save within wandb folder
                with self.model_path.open('wb') as model_file:
                    self.learn.save(model_file)

        # Log sample predictions if learn.predict is available
        if self.validation_data:
            try:
                self._wandb_log_predictions()
            except FastaiError as e:
                wandb.termwarn(e.message)
                self.validation_data = None  # prevent from trying again on next loop
            except Exception as e:
                wandb.termwarn("Unable to log prediction samples.\n{}".format(e))
                self.validation_data=None  # prevent from trying again on next loop

        # Log losses & metrics
        # Adapted from fast.ai "CSVLogger"
        logs = {
            name: stat
            for name, stat in list(
                zip(self.learn.recorder.names, [epoch, smooth_loss] +
                    last_metrics))
        }
        wandb.log(logs)

    def on_train_end(self, **kwargs):
        "Load the best model."

        if self.save_model:
            # Adapted from fast.ai "SaveModelCallback"
            if self.model_path.is_file():
                with self.model_path.open('rb') as model_file:
                    self.learn.load(model_file, purge=False)
                    print('Loaded best saved model from {}'.format(
                        self.model_path))

    def _wandb_log_predictions(self):
        "Log prediction samples"

        pred_log = []

        for x, y in self.validation_data:
            try:
                pred=self.learn.predict(x)
            except:
                raise FastaiError('Unable to run "predict" method from Learner to log prediction samples.')

            # scalar -> likely to be a category
            # tensor of dim 1 -> likely to be multicategory
            if not pred[1].shape or pred[1].dim() == 1:
                pred_log.append(
                    wandb.Image(
                        x.data,
                        caption='Ground Truth: {}\nPrediction: {}'.format(
                            y, pred[0])))

            # most vision datasets have a "show" function we can use
            elif hasattr(x, "show"):
                # log input data
                pred_log.append(
                    wandb.Image(x.data, caption='Input data', grouping=3))

                # log label and prediction
                for im, capt in ((pred[0], "Prediction"),
                                 (y, "Ground Truth")):
                    # Resize plot to image resolution
                    # from https://stackoverflow.com/a/13714915
                    my_dpi = 100
                    fig = plt.figure(frameon=False, dpi=my_dpi)
                    h, w = x.size
                    fig.set_size_inches(w / my_dpi, h / my_dpi)
                    ax = plt.Axes(fig, [0., 0., 1., 1.])
                    ax.set_axis_off()
                    fig.add_axes(ax)

                    # Superpose label or prediction to input image
                    x.show(ax=ax, y=im)
                    pred_log.append(wandb.Image(fig, caption=capt))
                    plt.close(fig)

            # likely to be an image
            elif hasattr(y, "shape") and (
                (len(y.shape) == 2) or
                    (len(y.shape) == 3 and y.shape[0] in [1, 3, 4])):

                pred_log.extend([
                    wandb.Image(x.data, caption='Input data', grouping=3),
                    wandb.Image(pred[0].data, caption='Prediction'),
                    wandb.Image(y.data, caption='Ground Truth')
                ])

            # we just log input data
            else:
                pred_log.append(wandb.Image(x.data, caption='Input data'))

            wandb.log({"Prediction Samples": pred_log}, commit=False)

class FastaiError(wandb.Error):
    pass
<|endoftext|>


<commit_before>from typing import Callable, Dict, Optional, Union

from six.moves import urllib
import wandb
from wandb import env
from wandb.apis import InternalApi
from wandb.util import handle_sweep_config_violations

from . import wandb_login

def _get_sweep_url(api, sweep_id):
    """Return sweep url if we can figure it out."""
    if api.api_key:
        if api.settings("entity") is None:
            viewer = api.viewer()
            if viewer.get("entity"):
                api.set_setting("entity", viewer["entity"])
        project = api.settings("project")
        if not project:
            return
        if api.settings("entity"):
            return "{base}/{entity}/{project}/sweeps/{sweepid}".format(
                base=api.app_url,
                entity=urllib.parse.quote(api.settings("entity")),
                project=urllib.parse.quote(project),
                sweepid=urllib.parse.quote(sweep_id),
            )

def sweep(
    sweep: Union[dict, Callable], entity: str = None, project: str = None,
) -> str:
    """Initialize a hyperparameter sweep.

    To generate hyperparameter suggestions from the sweep and use them
    to train a model, call `wandb.agent` with the sweep_id returned by
    this command. For command line functionality, see the command line
    tool `wandb sweep` (https://docs.wandb.ai/ref/cli/wandb-sweep).

    Args:
      sweep: dict, SweepConfig, or callable. The sweep configuration
        (or configuration generator). If a dict or SweepConfig,
        should conform to the W&B sweep config specification
        (https://docs.wandb.ai/guides/sweeps/configuration). If a
        callable, should take no arguments and return a dict that
        conforms to the W&B sweep config spec.
      entity: str (optional). An entity is a username or team name
        where you're sending runs. This entity must exist before you
        can send runs there, so make sure to create your account or
        team in the UI before starting to log runs.  If you don't
        specify an entity, the run will be sent to your default
        entity, which is usually your username. Change your default
        entity in [Settings](wandb.ai/settings) under "default
        location to create new projects".
      project: str (optional). The name of the project where you're
        sending the new run. If the project is not specified, the
        run is put in an "Uncategorized" project.

    Returns:
      sweep_id: str. A unique identifier for the sweep.

    Examples:
        Basic usage
        <!--yeadoc-test:one-parameter-sweep-->
        ```python
        import wandb
        sweep_configuration = {
            "name": "my-awesome-sweep",
            "metric": {"name": "accuracy", "goal": "maximize"},
            "method": "grid",
            "parameters": {
                "a": {
                    "values": [1, 2, 3, 4]
                }
            }
        }

        def my_train_func():
            # read the current value of parameter "a" from wandb.config
            wandb.init()
            a = wandb.config.a

            wandb.log({"a": a, "accuracy": a + 1})

        sweep_id = wandb.sweep(sweep_configuration)

        # run the sweep
        wandb.agent(sweep_id, function=my_train_func)
        ```
    """

    if callable(sweep):
        sweep = sweep()
    """Sweep create for controller api and jupyter (eventually for cli)."""
    if entity:
        env.set_entity(entity)
    if project:
        env.set_project(project)

    # Make sure we are logged in
    if wandb.run is None:
        wandb_login._login(_silent=True)
    api = InternalApi()
    sweep_id, warnings = api.upsert_sweep(sweep)
    handle_sweep_config_violations(warnings)
    print("Create sweep with ID:", sweep_id)
    sweep_url = _get_sweep_url(api, sweep_id)
    if sweep_url:
        print("Sweep URL:", sweep_url)
    return sweep_id

def controller(
    sweep_id_or_config: Optional[Union[str, Dict]] = None,
    entity: Optional[str] = None,
    project: Optional[str] = None,
):
    """Public sweep controller constructor.

    Usage:
        import wandb
        tuner = wandb.controller(...)
        print(tuner.sweep_config)
        print(tuner.sweep_id)
        tuner.configure_search(...)
        tuner.configure_stopping(...)

    """
    from ..wandb_controller import _WandbController

    c = _WandbController(
        sweep_id_or_config=sweep_id_or_config, entity=entity, project=project
    )
    return c
<commit_msg>Update a doctring in wandb_sweep.py (#3246)<commit_after>from typing import Callable, Dict, Optional, Union

from six.moves import urllib
import wandb
from wandb import env
from wandb.apis import InternalApi
from wandb.util import handle_sweep_config_violations

from . import wandb_login

def _get_sweep_url(api, sweep_id):
    """Return sweep url if we can figure it out."""
    if api.api_key:
        if api.settings("entity") is None:
            viewer = api.viewer()
            if viewer.get("entity"):
                api.set_setting("entity", viewer["entity"])
        project = api.settings("project")
        if not project:
            return
        if api.settings("entity"):
            return "{base}/{entity}/{project}/sweeps/{sweepid}".format(
                base=api.app_url,
                entity=urllib.parse.quote(api.settings("entity")),
                project=urllib.parse.quote(project),
                sweepid=urllib.parse.quote(sweep_id),
            )

def sweep(
    sweep: Union[dict, Callable], entity: str = None, project: str = None,
) -> str:
    """Initialize a hyperparameter sweep.

    To generate hyperparameter suggestions from the sweep and use them
    to train a model, call `wandb.agent` with the sweep_id returned by
    this command. For command line functionality, see the command line
    tool `wandb sweep` (https://docs.wandb.ai/ref/cli/wandb-sweep).

    Args:
      sweep: dict, SweepConfig, or callable. The sweep configuration
        (or configuration generator). If a dict or SweepConfig,
        should conform to the W&B sweep config specification
        (https://docs.wandb.ai/guides/sweeps/configuration). If a
        callable, should take no arguments and return a dict that
        conforms to the W&B sweep config spec.
      entity: str (optional). An entity is a username or team name
        where you're sending runs. This entity must exist before you
        can send runs there, so make sure to create your account or
        team in the UI before starting to log runs.  If you don't
        specify an entity, the run will be sent to your default
        entity, which is usually your username. Change your default
        entity in [Settings](wandb.ai/settings) under "default
        location to create new projects".
      project: str (optional). The name of the project where you're
        sending the new run. If the project is not specified, the
        run is put in an "Uncategorized" project.

    Returns:
      sweep_id: str. A unique identifier for the sweep.

    Examples:
        Basic usage
        <!--yeadoc-test:one-parameter-sweep-->
        ```python
        import wandb
        sweep_configuration = {
            "name": "my-awesome-sweep",
            "metric": {"name": "accuracy", "goal": "maximize"},
            "method": "grid",
            "parameters": {
                "a": {
                    "values": [1, 2, 3, 4]
                }
            }
        }

        def my_train_func():
            # read the current value of parameter "a" from wandb.config
            wandb.init()
            a = wandb.config.a

            wandb.log({"a": a, "accuracy": a + 1})

        sweep_id = wandb.sweep(sweep_configuration)

        # run the sweep
        wandb.agent(sweep_id, function=my_train_func)
        ```
    """

    if callable(sweep):
        sweep = sweep()
    """Sweep create for controller api and jupyter (eventually for cli)."""
    if entity:
        env.set_entity(entity)
    if project:
        env.set_project(project)

    # Make sure we are logged in
    if wandb.run is None:
        wandb_login._login(_silent=True)
    api = InternalApi()
    sweep_id, warnings = api.upsert_sweep(sweep)
    handle_sweep_config_violations(warnings)
    print("Create sweep with ID:", sweep_id)
    sweep_url = _get_sweep_url(api, sweep_id)
    if sweep_url:
        print("Sweep URL:", sweep_url)
    return sweep_id

def controller(
    sweep_id_or_config: Optional[Union[str, Dict]] = None,
    entity: Optional[str] = None,
    project: Optional[str] = None,
):
    """Public sweep controller constructor.

    Usage:
        ```python
        import wandb
        tuner = wandb.controller(...)
        print(tuner.sweep_config)
        print(tuner.sweep_id)
        tuner.configure_search(...)
        tuner.configure_stopping(...)
        ```

    """
    from ..wandb_controller import _WandbController

    c = _WandbController(
        sweep_id_or_config=sweep_id_or_config, entity=entity, project=project
    )
    return c
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from cogs.utils import checks
import asyncio
import clashroyale
import time
from datetime import datetime

try:
    from cogs.deck import Deck
except:
    print("Deck Cog not found, please do '!cog install Legend-Cogs deck'")

creditIcon = "https://i.imgur.com/TP8GXZb.png"
credits = "Bot by GR8 | Titan"

class warbattles:
    """Clash Royale Clan War attack log"""

    def __init__(self, bot):
        self.bot = bot
        self.auth = self.bot.get_cog('crtools').auth
        self.clans = self.bot.get_cog('crtools').clans
        self.constants = self.bot.get_cog('crtools').constants
        self.deck = Deck(self.bot)
        self.clash = clashroyale.OfficialAPI(self.auth.getOfficialToken(), is_async=True, timeout=20)
        self.moment = time.time()
        self.completed = [[], []]

    async def compareDeck(self, team, opp):
        """Check if both decks are similar"""
        deck1, deck2 = [], []

        for x in range(0, 8):
            deck1 = team[0].name
            deck2 = opp[0].name

        diff = len(set(deck1) - set(deck2))
        if diff > 3:
            return False
        return True

    async def getLevels(self, deck):
        """Get common, rare, epic, legendary levels"""
        levels = 0
        for card in deck:
            if card.max_level == 13:
                levels += card.level
            elif card.max_level == 11:
                levels += card.level + 3
            elif card.max_level == 8:
                levels += card.level + 6
            elif card.max_level == 5:
                levels += card.level + 9

        return levels

    async def deckStrength(self, team, opp):
        """Check if deck if underleveled or not"""
        perc = round((1 - (team / opp)) * 100, 2)
        return '{0:{1}}%'.format(perc, '+' if perc else '')

    async def cleanTime(self, time):
        """Converts time to timestamp"""
        return int(datetime.strptime(time, '%Y%m%dT%H%M%S.%fZ').timestamp()) + 7200

    async def get_clan_battles(self, clankey):
        """ Get war battles from each clan member"""
        battles = []
        for member in self.clans.keysClanMembers(clankey):
            battles += await self.clash.get_player_battles(member)
            await asyncio.sleep(0.08)
        return battles

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def warbattles(self, ctx):
        """War battle catcher with practice count, run in scedule"""
        for clankey in self.clans.keysClans():
            channel = await self.clans.getClanData(clankey, 'warlog_channel')
            if ((channel is not None) and (clankey not in self.completed[0])):
                try:
                    clanBattles = await self.get_clan_battles(clankey)
                except clashroyale.RequestError:
                    print("WARBATTLES: Cannot reach Clash Royale Servers.")
                    return

                for battle in clanBattles:
                    battledata = {"train": 0, "time": await self.cleanTime(battle.battle_time)}
                    if battledata["time"] > self.moment:
                        if battle.type == "clanWarWarDay" and (battledata["time"] not in self.completed[1]):
                            battledata["tag"] = battle.team[0].tag.strip("#")
                            battledata["name"] = battle.team[0].name
                            battledata["deckLink"] = await self.constants.decklink_url(battle.team[0].cards)
                            battledata["deckLevels"] = await self.deckStrength(await self.getLevels(battle.team[0].cards),
                                                                               await self.getLevels(battle.opponent[0].cards))
                            battledata["trophies"] = battle.opponent[0].starting_trophies - battle.team[0].starting_trophies

                            for pracBattle in clanBattles:
                                if pracBattle.team[0].tag.strip('#') == battledata["tag"]:
                                    if pracBattle.type not in ["clanWarWarDay", "PvP"]:
                                        pracTime = await self.cleanTime(pracBattle.battle_time)
                                        if (pracTime < battledata["time"]) and (self.moment - pracTime < 86400):
                                            if await self.compareDeck(pracBattle.team[0].cards, battle.team[0].cards):
                                                battledata["train"] += 1

                            battle.winner = battle.team[0].crowns - battle.opponent[0].crowns
                            if battle.winner >= 1:
                                battledata["wintext"] = "War Day Victory"
                                battledata["winicon"] = "https://royaleapi.com/static/img/ui/cw-war-win.png"
                                battledata["wincolor"] = discord.Color.green()
                            else:
                                battledata["wintext"] = "War Day Defeat"
                                battledata["winicon"] = "https://royaleapi.com/static/img/ui/cw-war-loss.png"
                                battledata["wincolor"] = discord.Color.red()

                            embed = discord.Embed(title="", description=battledata["wintext"], color=battledata["wincolor"])
                            embed.set_author(name=battledata["name"] + " (#"+battledata["tag"]+")",
                                             icon_url=await self.constants.get_clan_image(battle.team[0]))
                            embed.set_thumbnail(url=battledata["winicon"])
                            embed.add_field(name="Opponent Trophies",
                                            value='{0:{1}}'.format(battledata["trophies"], '+' if battledata["trophies"] else ''),
                                            inline=True)
                            embed.add_field(name="Opponent Card Levels", value=battledata["deckLevels"], inline=True)
                            embed.add_field(name="Practices", value=battledata["train"], inline=True)
                            embed.add_field(name="Deck Link",
                                            value="[Copy to war deck]({}&ID={}&war=1)".format(battledata["deckLink"],
                                                                                battledata["tag"]), inline=True)
                            embed.set_footer(text=credits, icon_url=creditIcon)

                            await self.bot.send_message(discord.Object(id=channel), embed=embed)

                            card_keys = await self.deck.decklink_to_cards(battledata["deckLink"])
                            newctx = ctx
                            newctx.message.channel = discord.Object(id=channel)
                            await self.deck.upload_deck_image(newctx, card_keys, 'War Deck')

                            self.completed[1].append(battledata["time"])
                self.completed[0].append(clankey)
                await asyncio.sleep(1)
        self.completed = [[], []]
        self.moment = time.time()

def setup(bot):
    bot.add_cog(warbattles(bot))
<commit_msg>fix card levels in warbattles<commit_after>import discord
from discord.ext import commands
from cogs.utils import checks
import asyncio
import clashroyale
import time
from datetime import datetime

try:
    from cogs.deck import Deck
except:
    print("Deck Cog not found, please do '!cog install Legend-Cogs deck'")

creditIcon = "https://i.imgur.com/TP8GXZb.png"
credits = "Bot by GR8 | Titan"

class warbattles:
    """Clash Royale Clan War attack log"""

    def __init__(self, bot):
        self.bot = bot
        self.auth = self.bot.get_cog('crtools').auth
        self.clans = self.bot.get_cog('crtools').clans
        self.constants = self.bot.get_cog('crtools').constants
        self.deck = Deck(self.bot)
        self.clash = clashroyale.OfficialAPI(self.auth.getOfficialToken(), is_async=True, timeout=20)
        self.moment = time.time()
        self.completed = [[], []]

    async def compareDeck(self, team, opp):
        """Check if both decks are similar"""
        deck1, deck2 = [], []

        for x in range(0, 8):
            deck1 = team[0].name
            deck2 = opp[0].name

        diff = len(set(deck1) - set(deck2))
        if diff > 3:
            return False
        return True

    async def getLevels(self, deck):
        """Get common, rare, epic, legendary levels"""
        levels = 0
        for card in deck:
            if card.max_level == 13:
                levels += card.level
            elif card.max_level == 11:
                levels += card.level + 2
            elif card.max_level == 8:
                levels += card.level + 5
            elif card.max_level == 5:
                levels += card.level + 8

        return levels

    async def deckStrength(self, team, opp):
        """Check if deck if underleveled or not"""
        perc = round((1 - (team / opp)) * 100, 2)
        return '{0:{1}}%'.format(perc, '+' if perc else '')

    async def cleanTime(self, time):
        """Converts time to timestamp"""
        return int(datetime.strptime(time, '%Y%m%dT%H%M%S.%fZ').timestamp()) + 7200

    async def get_clan_battles(self, clankey):
        """ Get war battles from each clan member"""
        battles = []
        for member in self.clans.keysClanMembers(clankey):
            battles += await self.clash.get_player_battles(member)
            await asyncio.sleep(0.08)
        return battles

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def warbattles(self, ctx):
        """War battle catcher with practice count, run in scedule"""
        for clankey in self.clans.keysClans():
            channel = await self.clans.getClanData(clankey, 'warlog_channel')
            if ((channel is not None) and (clankey not in self.completed[0])):
                try:
                    clanBattles = await self.get_clan_battles(clankey)
                except clashroyale.RequestError:
                    print("WARBATTLES: Cannot reach Clash Royale Servers.")
                    return

                for battle in clanBattles:
                    battledata = {"train": 0, "time": await self.cleanTime(battle.battle_time)}
                    if battledata["time"] > self.moment:
                        if battle.type == "clanWarWarDay" and (battledata["time"] not in self.completed[1]):
                            battledata["tag"] = battle.team[0].tag.strip("#")
                            battledata["name"] = battle.team[0].name
                            battledata["deckLink"] = await self.constants.decklink_url(battle.team[0].cards)
                            battledata["deckLevels"] = await self.deckStrength(await self.getLevels(battle.team[0].cards),
                                                                               await self.getLevels(battle.opponent[0].cards))
                            battledata["trophies"] = battle.opponent[0].starting_trophies - battle.team[0].starting_trophies

                            for pracBattle in clanBattles:
                                if pracBattle.team[0].tag.strip('#') == battledata["tag"]:
                                    if pracBattle.type not in ["clanWarWarDay", "PvP"]:
                                        pracTime = await self.cleanTime(pracBattle.battle_time)
                                        if (pracTime < battledata["time"]) and (self.moment - pracTime < 86400):
                                            if await self.compareDeck(pracBattle.team[0].cards, battle.team[0].cards):
                                                battledata["train"] += 1

                            battle.winner = battle.team[0].crowns - battle.opponent[0].crowns
                            if battle.winner >= 1:
                                battledata["wintext"] = "War Day Victory"
                                battledata["winicon"] = "https://royaleapi.com/static/img/ui/cw-war-win.png"
                                battledata["wincolor"] = discord.Color.green()
                            else:
                                battledata["wintext"] = "War Day Defeat"
                                battledata["winicon"] = "https://royaleapi.com/static/img/ui/cw-war-loss.png"
                                battledata["wincolor"] = discord.Color.red()

                            embed = discord.Embed(title="", description=battledata["wintext"], color=battledata["wincolor"])
                            embed.set_author(name=battledata["name"] + " (#"+battledata["tag"]+")",
                                             icon_url=await self.constants.get_clan_image(battle.team[0]))
                            embed.set_thumbnail(url=battledata["winicon"])
                            embed.add_field(name="Opponent Trophies",
                                            value='{0:{1}}'.format(battledata["trophies"], '+' if battledata["trophies"] else ''),
                                            inline=True)
                            embed.add_field(name="Opponent Card Levels", value=battledata["deckLevels"], inline=True)
                            embed.add_field(name="Practices", value=battledata["train"], inline=True)
                            embed.add_field(name="Deck Link",
                                            value="[Copy to war deck]({}&ID={}&war=1)".format(battledata["deckLink"],
                                                                                battledata["tag"]), inline=True)
                            embed.set_footer(text=credits, icon_url=creditIcon)

                            await self.bot.send_message(discord.Object(id=channel), embed=embed)

                            card_keys = await self.deck.decklink_to_cards(battledata["deckLink"])
                            newctx = ctx
                            newctx.message.channel = discord.Object(id=channel)
                            await self.deck.upload_deck_image(newctx, card_keys, 'War Deck')

                            self.completed[1].append(battledata["time"])
                self.completed[0].append(clankey)
                await asyncio.sleep(1)
        self.completed = [[], []]
        self.moment = time.time()

def setup(bot):
    bot.add_cog(warbattles(bot))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    snmp_simulator.simulator
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    SNMP Simulator
    :copyright: (c) 2014-2015 Dmitry Korobitsin <https://github.com/korobitsin>
    :license: BSD, see LICENSE
"""

import logging
import re
import time

from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdrsp, context
from pysnmp.carrier.asynsock.dgram import udp
from pysnmp.proto.api import v2c
from pysnmp.proto import rfc1902

from snmp_simulator.packages.lib_config import getConfig

log = logging.getLogger('simulator')
conf = getConfig('simulator')

TYPE_MAP = {
    'STRING': v2c.OctetString,
    'INTEGER': v2c.Integer32,
    'IpAddress': v2c.IpAddress,
    'Timeticks': v2c.TimeTicks,
    'Counter32': v2c.Counter32,
    'Counter64': v2c.Counter64,
    'OID': v2c.ObjectIdentifier,
    'Gauge32': v2c.Gauge32,
    'Hex-STRING': v2c.OctetString,
    'Network Address': v2c.OctetString,
}

def createVariable(SuperClass, getValue, *args):
    class Var(SuperClass):
        def getValue(self, name, idx):
            oid_value, oid_type_str = getValue(name, idx)
            return self.getSyntax().clone(TYPE_MAP[oid_type_str](oid_value))
    return Var(*args)

class SNMPAgent(object):
    def __init__(self, host, port, rcommunity):
        self.snmpEngine = engine.SnmpEngine()
        config.addSocketTransport(self.snmpEngine, udp.domainName, udp.UdpTransport().openServerMode((host, port)))
        config.addV1System(self.snmpEngine, 'my-area', rcommunity)
        config.addVacmUser(self.snmpEngine, 2, 'my-area', 'noAuthNoPriv', (1, 3, 6))
        config.addV3User(self.snmpEngine, 'test')
        config.addVacmUser(self.snmpEngine, 3, 'test', 'noAuthNoPriv', (1, 3, 6))
        self.snmpContext = context.SnmpContext(self.snmpEngine)
        self.mibBuilder = self.snmpContext.getMibInstrum().getMibBuilder()
        self.MibScalar, self.MibScalarInstance = self.mibBuilder.importSymbols('SNMPv2-SMI', 'MibScalar', 'MibScalarInstance')
        cmdrsp.GetCommandResponder(self.snmpEngine, self.snmpContext)
        cmdrsp.NextCommandResponder(self.snmpEngine, self.snmpContext)
        cmdrsp.BulkCommandResponder(self.snmpEngine, self.snmpContext)

    def add_oid(self, oid, oid_type_str, getValue):
        oid_num = rfc1902.ObjectName(oid).asTuple()
        oid_type = TYPE_MAP[oid_type_str]
        self.mibBuilder.exportSymbols(
            '__MY_MIB',
            self.MibScalar(oid_num[:-1], oid_type),
            createVariable(self.MibScalarInstance, getValue, oid_num[:-1], (oid_num[-1],), oid_type)
        )

    def serve_forever(self):
        self.snmpEngine.transportDispatcher.jobStarted(1)

        try:
            self.snmpEngine.transportDispatcher.runDispatcher()
        except KeyboardInterrupt:
            self.snmpEngine.transportDispatcher.closeDispatcher()

class Simulator(object):
    def __init__(self, host, port, rcommunity):
        self.start_time = time.time()
        self.oid_dict = {}
        self.snmp_agent = SNMPAgent(host=host, port=port, rcommunity=rcommunity)

    def add_walkfile(self, path):
        log.info('loading SNMP walk file {}'.format(path))
        with open(path) as f:
            try:
                file_text = f.read()
            except:
                log.exception('Failed to open walk file {}'.format(path))
                raise

        file_text = file_text.replace('\r', '')
        file_lines = file_text.split('\n')
        merged_lines = []
        for line in file_lines:
            if not line.startswith('.1.3.6') and merged_lines:
                merged_lines[-1] += '\n' + line
            else:
                merged_lines.append(line)

        reg_line = re.compile('^\.(.*?)\s*=\s*(.*)$', re.MULTILINE)
        reg_value = re.compile('^([a-zA-Z0-9\-\s]+?):\s*(.*)$')

        for line in merged_lines:
            try:
                reg_line_match = reg_line.match(line)
                oid, rest = reg_line_match.group(1), reg_line_match.group(2)
                reg_value_match = reg_value.match(rest)
                oid_type, oid_value = reg_value_match.group(1), reg_value_match.group(2)

                if oid_type in ('INTEGER', 'Gauge32', 'Gauge64', 'Counter32', 'Counter64'):
                    oid_value = int(oid_value)
                elif oid_type == 'STRING':
                    oid_value = oid_value.strip('"')
                elif oid_type == 'Timeticks':
                    oid_value = int(oid_value[1:].split(')')[0])

                self._add_oid_record({
                    'oid': oid,
                    'name': None,
                    'type': oid_type,
                    'value': oid_value,
                })
            except:
                log.debug('invalid line=%s' % line)
        log.info('loading SNMP walk file {} done!'.format(path))

    def _add_oid_record(self, record):
        self.oid_dict[record['oid']] = record
        self.snmp_agent.add_oid(record['oid'], record['type'], self.get_value)

    def get_value(self, name, oid_):
        oid = '.'.join([str(i) for i in name])
        oid_value, oid_type_str = self.oid_dict.get(oid, {}).get('value', 'Unknown value'), self.oid_dict.get(oid, {}).get('type', 'STRING')
        if oid == '1.3.6.1.2.1.1.3.0':
            oid_value = self.sysuptime()
        log.debug('get %s [%s] %s %s' % (oid, oid_, oid_type_str, oid_value))
        return oid_value, oid_type_str

    def sysuptime(self):
        # MIB2::sysUptime
        return int(time.time() - self.start_time)

def main():
    conf.script = 'simulator'
    conf.add_option(name='walk_file', desc='SNMP Walk file')
    conf.add_option(name='host', default='127.0.0.1')
    conf.add_option(name='port', default=161, otype=int)
    conf.add_option(name='rcommunity', default='public')
    conf.add_option(name='wcommunity', default='private')
    conf.setup(parse_cfg=False, pid=False)
    simulator = Simulator(host=conf.host, port=conf.port, rcommunity=conf.rcommunity)
    simulator.add_walkfile(conf.walk_file)
    simulator.snmp_agent.serve_forever()

if __name__ == "__main__":
    main()
<commit_msg>Fixed handling of Hex-STRING values<commit_after># -*- coding: utf-8 -*-
"""
    snmp_simulator.simulator
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    SNMP Simulator
    :copyright: (c) 2014-2015 Dmitry Korobitsin <https://github.com/korobitsin>
    :license: BSD, see LICENSE
"""

import logging
import re
import time

from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdrsp, context
from pysnmp.carrier.asynsock.dgram import udp
from pysnmp.proto.api import v2c
from pysnmp.proto import rfc1902

from snmp_simulator.packages.lib_config import getConfig

log = logging.getLogger('simulator')
conf = getConfig('simulator')

TYPE_MAP = {
    'STRING': v2c.OctetString,
    'INTEGER': v2c.Integer32,
    'IpAddress': v2c.IpAddress,
    'Timeticks': v2c.TimeTicks,
    'Counter32': v2c.Counter32,
    'Counter64': v2c.Counter64,
    'OID': v2c.ObjectIdentifier,
    'Gauge32': v2c.Gauge32,
    'Hex-STRING': v2c.OctetString,
    'Network Address': v2c.OctetString,
}

def createVariable(SuperClass, getValue, *args):
    class Var(SuperClass):
        def getValue(self, name, idx):
            oid_value, oid_type_str = getValue(name, idx)
            if oid_type_str == 'Hex-STRING':
                oid_value_kwarg = dict(hexValue=oid_value.replace(' ', ''))
            else:
                oid_value_kwarg = dict(value=oid_value)
            return self.getSyntax().clone(TYPE_MAP[oid_type_str](**oid_value_kwarg))
    return Var(*args)

class SNMPAgent(object):
    def __init__(self, host, port, rcommunity):
        self.snmpEngine = engine.SnmpEngine()
        config.addSocketTransport(self.snmpEngine, udp.domainName, udp.UdpTransport().openServerMode((host, port)))
        config.addV1System(self.snmpEngine, 'my-area', rcommunity)
        config.addVacmUser(self.snmpEngine, 2, 'my-area', 'noAuthNoPriv', (1, 3, 6))
        config.addV3User(self.snmpEngine, 'test')
        config.addVacmUser(self.snmpEngine, 3, 'test', 'noAuthNoPriv', (1, 3, 6))
        self.snmpContext = context.SnmpContext(self.snmpEngine)
        self.mibBuilder = self.snmpContext.getMibInstrum().getMibBuilder()
        self.MibScalar, self.MibScalarInstance = self.mibBuilder.importSymbols('SNMPv2-SMI', 'MibScalar', 'MibScalarInstance')
        cmdrsp.GetCommandResponder(self.snmpEngine, self.snmpContext)
        cmdrsp.NextCommandResponder(self.snmpEngine, self.snmpContext)
        cmdrsp.BulkCommandResponder(self.snmpEngine, self.snmpContext)

    def add_oid(self, oid, oid_type_str, getValue):
        oid_num = rfc1902.ObjectName(oid).asTuple()
        oid_type = TYPE_MAP[oid_type_str]
        self.mibBuilder.exportSymbols(
            '__MY_MIB',
            self.MibScalar(oid_num[:-1], oid_type),
            createVariable(self.MibScalarInstance, getValue, oid_num[:-1], (oid_num[-1],), oid_type)
        )

    def serve_forever(self):
        self.snmpEngine.transportDispatcher.jobStarted(1)

        try:
            self.snmpEngine.transportDispatcher.runDispatcher()
        except KeyboardInterrupt:
            self.snmpEngine.transportDispatcher.closeDispatcher()

class Simulator(object):
    def __init__(self, host, port, rcommunity):
        self.start_time = time.time()
        self.oid_dict = {}
        self.snmp_agent = SNMPAgent(host=host, port=port, rcommunity=rcommunity)

    def add_walkfile(self, path):
        log.info('loading SNMP walk file {}'.format(path))
        with open(path) as f:
            try:
                file_text = f.read()
            except:
                log.exception('Failed to open walk file {}'.format(path))
                raise

        file_text = file_text.replace('\r', '')
        file_lines = file_text.split('\n')
        merged_lines = []
        for line in file_lines:
            if not line.startswith('.1.3.6') and merged_lines:
                merged_lines[-1] += '\n' + line
            else:
                merged_lines.append(line)

        reg_line = re.compile('^\.(.*?)\s*=\s*(.*)$', re.MULTILINE)
        reg_value = re.compile('^([a-zA-Z0-9\-\s]+?):\s*(.*)$')

        for line in merged_lines:
            try:
                reg_line_match = reg_line.match(line)
                oid, rest = reg_line_match.group(1), reg_line_match.group(2)
                reg_value_match = reg_value.match(rest)
                oid_type, oid_value = reg_value_match.group(1), reg_value_match.group(2)

                if oid_type in ('INTEGER', 'Gauge32', 'Gauge64', 'Counter32', 'Counter64'):
                    oid_value = int(oid_value)
                elif oid_type == 'STRING':
                    oid_value = oid_value.strip('"')
                elif oid_type == 'Timeticks':
                    oid_value = int(oid_value[1:].split(')')[0])

                self._add_oid_record({
                    'oid': oid,
                    'name': None,
                    'type': oid_type,
                    'value': oid_value,
                })
            except:
                log.debug('invalid line=%s' % line)
        log.info('loading SNMP walk file {} done!'.format(path))

    def _add_oid_record(self, record):
        self.oid_dict[record['oid']] = record
        self.snmp_agent.add_oid(record['oid'], record['type'], self.get_value)

    def get_value(self, name, oid_):
        oid = '.'.join([str(i) for i in name])
        oid_value, oid_type_str = self.oid_dict.get(oid, {}).get('value', 'Unknown value'), self.oid_dict.get(oid, {}).get('type', 'STRING')
        if oid == '1.3.6.1.2.1.1.3.0':
            oid_value = self.sysuptime()
        log.debug('get %s [%s] %s %s' % (oid, oid_, oid_type_str, oid_value))
        return oid_value, oid_type_str

    def sysuptime(self):
        # MIB2::sysUptime
        return int(time.time() - self.start_time)

def main():
    conf.script = 'simulator'
    conf.add_option(name='walk_file', desc='SNMP Walk file')
    conf.add_option(name='host', default='127.0.0.1')
    conf.add_option(name='port', default=161, otype=int)
    conf.add_option(name='rcommunity', default='public')
    conf.add_option(name='wcommunity', default='private')
    conf.setup(parse_cfg=False, pid=False)
    simulator = Simulator(host=conf.host, port=conf.port, rcommunity=conf.rcommunity)
    simulator.add_walkfile(conf.walk_file)
    simulator.snmp_agent.serve_forever()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os

class WaterButlerPathPart:
    DECODE = lambda x: x
    ENCODE = lambda x: x

    def __init__(self, part, _id=None):
        self._id = _id
        self._part = part

    @property
    def identifier(self):
        return self._id

    @property
    def value(self):
        return self.__class__.DECODE(self._part)

    @property
    def raw(self):
        return self._part

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self._part)

class WaterButlerPath:
    """
    A standardized and validated immutable WaterButler path.
    """

    PART_CLASS = WaterButlerPathPart

    @classmethod
    def generic_path_validation(cls, path):
        """Validates a WaterButler specific path, e.g. /folder/file.txt, /folder/
        :param str path: WaterButler path
        """
        if not path:
            raise exceptions.InvalidPathError('Must specify path')
        if not path.startswith('/'):
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(path))
        if '//' in path:
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(path))
        # Do not allow path manipulation via shortcuts, e.g. '..'
        absolute_path = os.path.abspath(path)
        if not path == '/' and path.endswith('/'):
            absolute_path += '/'
        if not path == absolute_path:
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(absolute_path))

    @classmethod
    def from_parts(cls, parts, folder=False):
        _ids, _parts = [], ['']
        for part in parts:
            _ids.append(part.identifier)
            _parts.append(part.raw)

        return cls('/'.join(_parts), _ids=_ids, folder=folder)

    def __init__(self, path, _ids=(), prepend=None, folder=None):
        self.__class__.generic_path_validation(path)

        self._orig_path = path

        self._prepend = prepend

        if prepend:
            self._prepend_parts = [self.PART_CLASS(part, None) for part in prepend.rstrip('/').split('/')]
        else:
            self._prepend_parts = []

        self._parts = [
            self.PART_CLASS(part, _id)
            for _id, part in
            itertools.zip_longest(_ids, path.rstrip('/').split('/'))
        ]

        if folder is not None:
            self._is_folder = bool(folder)
        else:
            self._is_folder = self._orig_path.endswith('/')

        if self.is_dir and not self._orig_path.endswith('/'):
            self._orig_path += '/'

    @property
    def is_root(self):
        return len(self._parts) == 1

    @property
    def is_dir(self):
        return self._is_folder

    @property
    def is_file(self):
        return not self._is_folder

    @property
    def parts(self):
        return self._parts

    @property
    def name(self):
        return self._parts[-1].value

    @property
    def identifier(self):
        return self._parts[-1].identifier

    @property
    def path(self):
        return '/'.join([x.value for x in self.parts[1:]]) + ('/' if self.is_dir else '')

    @property
    def full_path(self):
        return '/'.join([x.value for x in self._prepend_parts + self.parts[1:]]) + ('/' if self.is_dir else '')

    @property
    def parent(self):
        if len(self.parts) == 1:
            return None
        return self.__class__.from_parts(self.parts[:-1], folder=True, prepend=self._prepend)

    def __str__(self):
        return '/'.join([x.raw for x in self.parts]) + ('/' if self.is_dir else '')

    def __repr__(self):
        return '{}({!r}, prepend={!r})'.format(self.__class__.__name__, self._orig_path, self._prepend)
<commit_msg>Add helpers for directory checking<commit_after>import os

class WaterButlerPathPart:
    DECODE = lambda x: x
    ENCODE = lambda x: x

    def __init__(self, part, _id=None):
        self._id = _id
        self._part = part

    @property
    def identifier(self):
        return self._id

    @property
    def value(self):
        return self.__class__.DECODE(self._part)

    @property
    def raw(self):
        return self._part

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self._part)

class WaterButlerPath:
    """
    A standardized and validated immutable WaterButler path.
    """

    PART_CLASS = WaterButlerPathPart

    @classmethod
    def generic_path_validation(cls, path):
        """Validates a WaterButler specific path, e.g. /folder/file.txt, /folder/
        :param str path: WaterButler path
        """
        if not path:
            raise exceptions.InvalidPathError('Must specify path')
        if not path.startswith('/'):
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(path))
        if '//' in path:
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(path))
        # Do not allow path manipulation via shortcuts, e.g. '..'
        absolute_path = os.path.abspath(path)
        if not path == '/' and path.endswith('/'):
            absolute_path += '/'
        if not path == absolute_path:
            raise exceptions.InvalidPathError('Invalid path \'{}\' specified'.format(absolute_path))

    @classmethod
    def validate_folder(cls, path):
        if not path.is_dir:
            raise exceptions.CreateFolderError('Path must be a directory', code=400)

        if path.is_root:
            raise exceptions.CreateFolderError('Path can not be root', code=400)

    @classmethod
    def from_parts(cls, parts, folder=False):
        _ids, _parts = [], ['']
        for part in parts:
            _ids.append(part.identifier)
            _parts.append(part.raw)

        return cls('/'.join(_parts), _ids=_ids, folder=folder)

    def __init__(self, path, _ids=(), prepend=None, folder=None):
        self.__class__.generic_path_validation(path)

        self._orig_path = path

        self._prepend = prepend

        if prepend:
            self._prepend_parts = [self.PART_CLASS(part, None) for part in prepend.rstrip('/').split('/')]
        else:
            self._prepend_parts = []

        self._parts = [
            self.PART_CLASS(part, _id)
            for _id, part in
            itertools.zip_longest(_ids, path.rstrip('/').split('/'))
        ]

        if folder is not None:
            self._is_folder = bool(folder)
        else:
            self._is_folder = self._orig_path.endswith('/')

        if self.is_dir and not self._orig_path.endswith('/'):
            self._orig_path += '/'

    @property
    def is_root(self):
        return len(self._parts) == 1

    @property
    def is_dir(self):
        return self._is_folder

    @property
    def is_file(self):
        return not self._is_folder

    @property
    def parts(self):
        return self._parts

    @property
    def name(self):
        return self._parts[-1].value

    @property
    def identifier(self):
        return self._parts[-1].identifier

    @property
    def path(self):
        return '/'.join([x.value for x in self.parts[1:]]) + ('/' if self.is_dir else '')

    @property
    def full_path(self):
        return '/'.join([x.value for x in self._prepend_parts + self.parts[1:]]) + ('/' if self.is_dir else '')

    @property
    def parent(self):
        if len(self.parts) == 1:
            return None
        return self.__class__.from_parts(self.parts[:-1], folder=True, prepend=self._prepend)

    def __str__(self):
        return '/'.join([x.raw for x in self.parts]) + ('/' if self.is_dir else '')

    def __repr__(self):
        return '{}({!r}, prepend={!r})'.format(self.__class__.__name__, self._orig_path, self._prepend)
<|endoftext|>