<commit_before>"""
Just-in-time compilation support.
"""

import abc
import copy
import shutil
import tempfile
import os
import hashlib
import string
import re

import ctree
from ctree.nodes import Project
from ctree.analyses import VerifyOnlyCtreeNodes
from ctree.util import highlight
from ctree.frontend import get_ast

import ast

import llvm.core as ll

import logging
import inspect
import hashlib
import json

from ctree.c.nodes import CFile, FunctionDecl, FunctionCall, MultiNode
from ctree.ocl.nodes import OclFile
from ctree.nodes import File

from collections import namedtuple

import itertools

log = logging.getLogger(__name__)

def getFile(filepath):
    """
    Takes a filepath and returns a specialized File instance (i.e. OclFile, CFile, etc)
    """
    ext_map = {'.'+t._ext:t for t in (
        CFile, OclFile
    )}
    path, filename = os.path.split(filepath)
    name, ext = os.path.splitext(filename)
    filetype = ext_map[ext]
    return filetype(name=name, path=path)

class JitModule(object):
    """
    Manages compilation of multiple ASTs.
    """

    def __init__(self):
        '''compilation_dir specifies the name of the subfolder under COMPILE_PATH'''
        # write files to $TEMPDIR/ctree/run-XXXX
        # compile_to = os.path.expanduser(ctree.CONFIG.get('jit','COMPILE_PATH'))
        #
        # # makes sure that directories exists, otherwise creates
        # if not compile_to:
        #     compile_to = os.path.join(tempfile.gettempdir(), "ctree")
        #
        # if compilation_dir:
        #     self.compilation_dir = os.path.join(compile_to, compilation_dir)
        # else:
        #     self.compilation_dir = tempfile.mkdtemp(prefix="run-", dir=compile_to)
        # if not os.path.exists(self.compilation_dir):
        #     os.makedirs(self.compilation_dir)
        #
        # log.info('compiling to %s'%self.compilation_dir)
        self.ll_module = ll.Module.new('ctree')
        self.exec_engine = None
        # log.info("temporary compilation directory is: %s",
        #          self.compilation_dir)

    # def __del__(self):
    #     if not ctree.CONFIG.get("jit", "PRESERVE_SRC_DIR"):
    #         log.info("removing temporary compilation directory %s.",
    #                  self.compilation_dir)
    #         shutil.rmtree(self.compilation_dir)

    def _link_in(self, submodule):
        self.ll_module.link_in(submodule)

    def get_callable(self, entry_point_name, entry_point_typesig):
        """
        Returns a python callable that dispatches to the requested C function.
        """

        # get llvm represetation of function
        ll_function = self.ll_module.get_function_named(entry_point_name)

        # run jit compiler
        from llvm.ee import EngineBuilder

        self.exec_engine = \
            EngineBuilder.new(self.ll_module).mcjit(True).opt(3).create()

        c_func_ptr = self.exec_engine.get_pointer_to_function(ll_function)

        # cast c_func_ptr to python callable using ctypes
        return entry_point_typesig(c_func_ptr)

class ConcreteSpecializedFunction(object):
    """
    A function backed by generated code.
    """
    __metaclass__ = abc.ABCMeta

    def _compile(self, entry_point_name, project_node, entry_point_typesig,
                 **kwargs):
        """
        Returns a python callable.
        """
        assert isinstance(project_node, Project), \
            "Expected a Project but it got a %s." % type(project_node)

        VerifyOnlyCtreeNodes().visit(project_node)

        self._module = project_node.codegen(**kwargs)

        highlighted = highlight(str(self._module.ll_module), 'llvm')
        log.debug("full LLVM program is: <<<\n%s\n>>>" % highlighted)

        return self._module.get_callable(entry_point_name, entry_point_typesig)

    @abc.abstractmethod
    def __call__(self, *args, **kwargs):
        pass

class LazySpecializedFunction(object):
    """
    A callable object that will produce executable
    code just-in-time.
    """

    ProgramConfig = namedtuple('ProgramConfig',['args_subconfig', 'tuner_subconfig'])

    def __init__(self, py_ast = None):
        if py_ast is not None:
            raise TypeError('This functionality has been removed and the signature will be modified in future versions')
        self.original_tree = py_ast or get_ast(self.apply)
        self.concrete_functions = {}  # config -> callable map
        self._tuner = self.get_tuning_driver()

    @property
    def info_filename(self):
        return 'info.json'

    def get_info(self, path):
        info_filepath = os.path.join(path, self.info_filename)
        if not os.path.exists(info_filepath):
            return {'hash':None, 'files':[]}
        with open(info_filepath) as info_file:
            return json.load(info_file)

    def set_info(self, path, dictionary):
        info_filepath = os.path.join(path, self.info_filename)
        with open(info_filepath,'w') as info_file:
            return json.dump(dictionary, info_file)

    @staticmethod
    def _hash(o):
        if isinstance(o, dict):
            return hash(frozenset(
                LazySpecializedFunction._hash(item) for item in o.items()
            ))
        else:
            return hash(str(o))

    def __hash__(self):
        mro = type(self).mro()
        result = hashlib.sha512('')
        for klass in mro:
            if issubclass(klass, LazySpecializedFunction):
                result.update(inspect.getsource(klass).encode())
            else:
                pass
        return int(result.hexdigest(), 16)

    def config_to_dirname(self, program_config):
        """Returns the subdirectory name under .compiled/funcname"""
        # fixes the directory names and squishes invalid chars
        forbidden_chars = r"""/\?%*:|"<>()' """
        config_str = ''.join(i for i in str(program_config) if i not in forbidden_chars)
        config_path = re.sub("_+","_", config_str)
        path = os.path.join(ctree.CONFIG.get('jit','COMPILE_PATH'),self.__class__.__name__, config_path)
        return path

    def __call__(self, *args, **kwargs):
        """
            Determines the program_configuration to be run. If it has yet to be
            built, build it. Then, execute it. If the selected program_configuration
            for this function has already been code generated for, this method draws
            from the cache.
        """
        ctree.STATS.log("specialized function call")
        assert not kwargs, \
            "Passing kwargs to specialized functions isn't supported."

        log.info("detected specialized function call with arg types: %s",
                 [type(a) for a in args])

        args_subconfig = self.args_to_subconfig(args)
        tuner_subconfig = next(self._tuner.configs)
        program_config = self.ProgramConfig(args_subconfig, tuner_subconfig)
        dir_name = self.config_to_dirname(program_config)
        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

        log.info("tuner subconfig: %s", tuner_subconfig)
        log.info("arguments subconfig: %s", args_subconfig)

        config_hash = dir_name

        if config_hash in self.concrete_functions:              # checks to see if the necessary code is in the run-time cache
            ctree.STATS.log("specialized function cache hit")
            log.info("specialized function cache hit!")
            csf = self.concrete_functions[config_hash]

        else:
            ctree.STATS.log("specialized function cache miss")
            log.info("specialized function cache miss.")
            info = self.get_info(dir_name)
            if hash(self) != info['hash']:                      # checks to see if the necessary code is in the persistent cache

                # need to run transform() for code generation
                log.info('Hash miss. Running Transform')
                ctree.STATS.log("Filesystem cache miss")
                transform_result = self.transform(
                    copy.deepcopy(self.original_tree),          # TODO: is this deepcopy really necessary?
                    program_config
                )
                if not isinstance(transform_result, (tuple, list)):
                    transform_result = (transform_result,)
                for source_file in transform_result:
                    assert isinstance(source_file, File), "Transform must return an iterable of Files"
                    source_file.path = dir_name
                new_info = {'hash': hash(self), 'files':[os.path.join(f.path, f.get_filename()) for f in transform_result]}
                self.set_info(dir_name, new_info)

            else:
                log.info('Hash hit. Skipping transform')
                ctree.STATS.log('Filesystem cache hit')
                files = [getFile(path) for path in info['files']]
                transform_result = files

            csf = self.finalize(transform_result, program_config)
            assert isinstance(csf, ConcreteSpecializedFunction), "Expected a ctree.jit.ConcreteSpecializedFunction, but got a %s." % type(csf)
            self.concrete_functions[config_hash] = csf
        return csf(*args, **kwargs)

    @classmethod
    def from_function(cls, func, classname = ''):
        class Replacer(ast.NodeTransformer):
            def visit_Module(self, node):
                return MultiNode(body = [self.visit(i) for i in node.body])

            def visit_FunctionDef(self, node):
                if node.name == func.__name__:
                    node.name = 'apply'
                node.body = [self.visit(item) for item in node.body]
                return node

            def visit_Name(self, node):
                if node.id == func.__name__:
                    node.id = 'apply'
                return node

        def transform(self, tree, program_config):
            """
                Calls transform after renaming the function name to 'apply' since specializers are written assuming "apply"
            """
            tree = Replacer().visit(tree)
            return super(newClass, self).transform(tree, program_config)

        def __hash__(self):
            func_hash = int(hashlib.sha512(inspect.getsource(func).encode()).hexdigest(), 16)
            old_hash = hash(cls())
            return func_hash ^ old_hash
        newClass = type(classname or func.__name__, (cls, ), {'apply': staticmethod(func), '__hash__':
                                                 __hash__,
                                                 'transform': transform
                                                })

        return newClass

    def report(self, *args, **kwargs):
        """
        Records the performance of the most recent configuration.
        """
        return self._tuner.report(*args, **kwargs)

    # =====================================================
    # Methods to be overridden by the user

    def transform(self, tree, program_config):
        """
        Convert the AST 'tree' into a C AST, optionally taking advantage of the
        actual runtime arguments.
        """
        raise NotImplementedError()

    def finalize(self, transform_result, program_config):
        """
        This function will be passed the result of transform.  The specializer
        should return an ConcreteSpecializedFunction.
        """
        raise NotImplementedError("Finalize must be implemented")

    def get_tuning_driver(self):
        """
        Define the space of possible implementations.
        """
        from ctree.tune import ConstantTuningDriver

        return ConstantTuningDriver()

    def args_to_subconfig(self, args):
        """
        Extract features from the arguments to define uniqueness of
        this particular invocation. The return value must be a hashable
        object, or a dictionary of hashable objects.
        """
        log.warn("arguments will not influence program_config. " +
                 "Consider overriding args_to_subconfig() in %s.",
                 type(self).__name__)
        return dict()

    @staticmethod
    def apply(*args):
        raise NotImplementedError()
<commit_msg>apparently python thinks everything is unicode in py3k, even empty strings. Had to add another str.encode call to make hashlib work.<commit_after>"""
Just-in-time compilation support.
"""

import abc
import copy
import shutil
import tempfile
import os
import hashlib
import string
import re

import ctree
from ctree.nodes import Project
from ctree.analyses import VerifyOnlyCtreeNodes
from ctree.util import highlight
from ctree.frontend import get_ast

import ast

import llvm.core as ll

import logging
import inspect
import hashlib
import json

from ctree.c.nodes import CFile, FunctionDecl, FunctionCall, MultiNode
from ctree.ocl.nodes import OclFile
from ctree.nodes import File

from collections import namedtuple

import itertools

log = logging.getLogger(__name__)

def getFile(filepath):
    """
    Takes a filepath and returns a specialized File instance (i.e. OclFile, CFile, etc)
    """
    ext_map = {'.'+t._ext:t for t in (
        CFile, OclFile
    )}
    path, filename = os.path.split(filepath)
    name, ext = os.path.splitext(filename)
    filetype = ext_map[ext]
    return filetype(name=name, path=path)

class JitModule(object):
    """
    Manages compilation of multiple ASTs.
    """

    def __init__(self):
        '''compilation_dir specifies the name of the subfolder under COMPILE_PATH'''
        # write files to $TEMPDIR/ctree/run-XXXX
        # compile_to = os.path.expanduser(ctree.CONFIG.get('jit','COMPILE_PATH'))
        #
        # # makes sure that directories exists, otherwise creates
        # if not compile_to:
        #     compile_to = os.path.join(tempfile.gettempdir(), "ctree")
        #
        # if compilation_dir:
        #     self.compilation_dir = os.path.join(compile_to, compilation_dir)
        # else:
        #     self.compilation_dir = tempfile.mkdtemp(prefix="run-", dir=compile_to)
        # if not os.path.exists(self.compilation_dir):
        #     os.makedirs(self.compilation_dir)
        #
        # log.info('compiling to %s'%self.compilation_dir)
        self.ll_module = ll.Module.new('ctree')
        self.exec_engine = None
        # log.info("temporary compilation directory is: %s",
        #          self.compilation_dir)

    # def __del__(self):
    #     if not ctree.CONFIG.get("jit", "PRESERVE_SRC_DIR"):
    #         log.info("removing temporary compilation directory %s.",
    #                  self.compilation_dir)
    #         shutil.rmtree(self.compilation_dir)

    def _link_in(self, submodule):
        self.ll_module.link_in(submodule)

    def get_callable(self, entry_point_name, entry_point_typesig):
        """
        Returns a python callable that dispatches to the requested C function.
        """

        # get llvm represetation of function
        ll_function = self.ll_module.get_function_named(entry_point_name)

        # run jit compiler
        from llvm.ee import EngineBuilder

        self.exec_engine = \
            EngineBuilder.new(self.ll_module).mcjit(True).opt(3).create()

        c_func_ptr = self.exec_engine.get_pointer_to_function(ll_function)

        # cast c_func_ptr to python callable using ctypes
        return entry_point_typesig(c_func_ptr)

class ConcreteSpecializedFunction(object):
    """
    A function backed by generated code.
    """
    __metaclass__ = abc.ABCMeta

    def _compile(self, entry_point_name, project_node, entry_point_typesig,
                 **kwargs):
        """
        Returns a python callable.
        """
        assert isinstance(project_node, Project), \
            "Expected a Project but it got a %s." % type(project_node)

        VerifyOnlyCtreeNodes().visit(project_node)

        self._module = project_node.codegen(**kwargs)

        highlighted = highlight(str(self._module.ll_module), 'llvm')
        log.debug("full LLVM program is: <<<\n%s\n>>>" % highlighted)

        return self._module.get_callable(entry_point_name, entry_point_typesig)

    @abc.abstractmethod
    def __call__(self, *args, **kwargs):
        pass

class LazySpecializedFunction(object):
    """
    A callable object that will produce executable
    code just-in-time.
    """

    ProgramConfig = namedtuple('ProgramConfig',['args_subconfig', 'tuner_subconfig'])

    def __init__(self, py_ast = None):
        if py_ast is not None:
            raise TypeError('This functionality has been removed and the signature will be modified in future versions')
        self.original_tree = py_ast or get_ast(self.apply)
        self.concrete_functions = {}  # config -> callable map
        self._tuner = self.get_tuning_driver()

    @property
    def info_filename(self):
        return 'info.json'

    def get_info(self, path):
        info_filepath = os.path.join(path, self.info_filename)
        if not os.path.exists(info_filepath):
            return {'hash':None, 'files':[]}
        with open(info_filepath) as info_file:
            return json.load(info_file)

    def set_info(self, path, dictionary):
        info_filepath = os.path.join(path, self.info_filename)
        with open(info_filepath,'w') as info_file:
            return json.dump(dictionary, info_file)

    @staticmethod
    def _hash(o):
        if isinstance(o, dict):
            return hash(frozenset(
                LazySpecializedFunction._hash(item) for item in o.items()
            ))
        else:
            return hash(str(o))

    def __hash__(self):
        mro = type(self).mro()
        result = hashlib.sha512(''.encode())
        for klass in mro:
            if issubclass(klass, LazySpecializedFunction):
                result.update(inspect.getsource(klass).encode())
            else:
                pass
        return int(result.hexdigest(), 16)

    def config_to_dirname(self, program_config):
        """Returns the subdirectory name under .compiled/funcname"""
        # fixes the directory names and squishes invalid chars
        forbidden_chars = r"""/\?%*:|"<>()' """
        config_str = ''.join(i for i in str(program_config) if i not in forbidden_chars)
        config_path = re.sub("_+","_", config_str)
        path = os.path.join(ctree.CONFIG.get('jit','COMPILE_PATH'),self.__class__.__name__, config_path)
        return path

    def __call__(self, *args, **kwargs):
        """
            Determines the program_configuration to be run. If it has yet to be
            built, build it. Then, execute it. If the selected program_configuration
            for this function has already been code generated for, this method draws
            from the cache.
        """
        ctree.STATS.log("specialized function call")
        assert not kwargs, \
            "Passing kwargs to specialized functions isn't supported."

        log.info("detected specialized function call with arg types: %s",
                 [type(a) for a in args])

        args_subconfig = self.args_to_subconfig(args)
        tuner_subconfig = next(self._tuner.configs)
        program_config = self.ProgramConfig(args_subconfig, tuner_subconfig)
        dir_name = self.config_to_dirname(program_config)
        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

        log.info("tuner subconfig: %s", tuner_subconfig)
        log.info("arguments subconfig: %s", args_subconfig)

        config_hash = dir_name

        if config_hash in self.concrete_functions:              # checks to see if the necessary code is in the run-time cache
            ctree.STATS.log("specialized function cache hit")
            log.info("specialized function cache hit!")
            csf = self.concrete_functions[config_hash]

        else:
            ctree.STATS.log("specialized function cache miss")
            log.info("specialized function cache miss.")
            info = self.get_info(dir_name)
            if hash(self) != info['hash']:                      # checks to see if the necessary code is in the persistent cache

                # need to run transform() for code generation
                log.info('Hash miss. Running Transform')
                ctree.STATS.log("Filesystem cache miss")
                transform_result = self.transform(
                    copy.deepcopy(self.original_tree),          # TODO: is this deepcopy really necessary?
                    program_config
                )
                if not isinstance(transform_result, (tuple, list)):
                    transform_result = (transform_result,)
                for source_file in transform_result:
                    assert isinstance(source_file, File), "Transform must return an iterable of Files"
                    source_file.path = dir_name
                new_info = {'hash': hash(self), 'files':[os.path.join(f.path, f.get_filename()) for f in transform_result]}
                self.set_info(dir_name, new_info)

            else:
                log.info('Hash hit. Skipping transform')
                ctree.STATS.log('Filesystem cache hit')
                files = [getFile(path) for path in info['files']]
                transform_result = files

            csf = self.finalize(transform_result, program_config)
            assert isinstance(csf, ConcreteSpecializedFunction), "Expected a ctree.jit.ConcreteSpecializedFunction, but got a %s." % type(csf)
            self.concrete_functions[config_hash] = csf
        return csf(*args, **kwargs)

    @classmethod
    def from_function(cls, func, classname = ''):
        class Replacer(ast.NodeTransformer):
            def visit_Module(self, node):
                return MultiNode(body = [self.visit(i) for i in node.body])

            def visit_FunctionDef(self, node):
                if node.name == func.__name__:
                    node.name = 'apply'
                node.body = [self.visit(item) for item in node.body]
                return node

            def visit_Name(self, node):
                if node.id == func.__name__:
                    node.id = 'apply'
                return node

        def transform(self, tree, program_config):
            """
                Calls transform after renaming the function name to 'apply' since specializers are written assuming "apply"
            """
            tree = Replacer().visit(tree)
            return super(newClass, self).transform(tree, program_config)

        def __hash__(self):
            func_hash = int(hashlib.sha512(inspect.getsource(func).encode()).hexdigest(), 16)
            old_hash = hash(cls())
            return func_hash ^ old_hash
        newClass = type(classname or func.__name__, (cls, ), {'apply': staticmethod(func), '__hash__':
                                                 __hash__,
                                                 'transform': transform
                                                })

        return newClass

    def report(self, *args, **kwargs):
        """
        Records the performance of the most recent configuration.
        """
        return self._tuner.report(*args, **kwargs)

    # =====================================================
    # Methods to be overridden by the user

    def transform(self, tree, program_config):
        """
        Convert the AST 'tree' into a C AST, optionally taking advantage of the
        actual runtime arguments.
        """
        raise NotImplementedError()

    def finalize(self, transform_result, program_config):
        """
        This function will be passed the result of transform.  The specializer
        should return an ConcreteSpecializedFunction.
        """
        raise NotImplementedError("Finalize must be implemented")

    def get_tuning_driver(self):
        """
        Define the space of possible implementations.
        """
        from ctree.tune import ConstantTuningDriver

        return ConstantTuningDriver()

    def args_to_subconfig(self, args):
        """
        Extract features from the arguments to define uniqueness of
        this particular invocation. The return value must be a hashable
        object, or a dictionary of hashable objects.
        """
        log.warn("arguments will not influence program_config. " +
                 "Consider overriding args_to_subconfig() in %s.",
                 type(self).__name__)
        return dict()

    @staticmethod
    def apply(*args):
        raise NotImplementedError()
<|endoftext|>


<commit_before>import sys
import struct
import os
import argparse
import json
import re

def readInt32(file):
    value = file.read(4)
    if not value:
        return
    return struct.unpack('>i', value)[0]

def readInt64(file):
    value = file.read(8)
    if not value:
        return
    return struct.unpack('>q', value)[0]

def readByte(file):
    return struct.unpack('>B', file.read(1))[0]

def readString(file):
    length = readInt32(file)
    return struct.unpack('%ds' % length, file.read(length))[0].decode('utf-8')

def secondsToString(seconds):
    hours = int(seconds / 3600)
    seconds %= 3600
    minutes = int(seconds / 60)
    seconds %= 60

    str = ""
    if hours > 0:
        str += "%dh " % hours
    if hours == 0 and minutes == 0:
        str += "%ds" % seconds
    else:
        str += "%dm" % minutes

    return str

def searchPlayer(id, clan):
    return [player for player in clan['roster'] if player['id'] == id][0]

def swapAttackOrder(clan):
    for player in clan['roster']:
        if player['attacksUsed'] == 1:
            temp = player['attack1']
            player['attack1'] = player['attack2']
            player['attack2'] = temp

# Print raw output
def printList(values):
    for v in values:
        print(v, end = ',', file = fileOut)
    print("", file = fileOut)

def parseClan(file):
    values = []
    clan = {}

    values.append(readInt64(file))  # Clan ID
    clan['id'] = values[-1]

    values.append(readString(file)) # Clan name
    clan['name'] = values[-1]

    values.append(readInt32(file))  # Clan badge

    values.append(readInt32(file))  # Clan level
    clan['level'] = values[-1]

    values.append(readInt32(file))  # War size
    clan['size'] = values[-1]
    warSize = values[-1]

    print(clanLabel, file = fileOut)
    printList(values)

    print(memberLabel, file = fileOut)

    roster = []
    for i in range(0, warSize):
        player = parseMember(file)
        player['position'] = i + 1
        roster.append(player)

    clan['roster'] = roster
    return clan

def parseMember(file):
    values = [] # raw output
    player = {} # formatted output

    player['attacksUsed'] = 0
    player['attacksWon'] = 0
    player['attack1'] = {'starsWon': 0, 'starsEarned': 0, 'damage': 0, 'target': "", 'targetPosition': 0}
    player['attack2'] = {'starsWon': 0, 'starsEarned': 0, 'damage': 0, 'target': "", 'targetPosition': 0}

    values.append(readInt64(file))  # Clan ID

    values.append(readInt64(file))  # Member ID
    player['id'] = values[-1]

    values.append(readInt64(file))  # Member ID

    values.append(readString(file)) # Member Name
    player['name'] = values[-1]
    values[-1] = '"' + values[-1] + '"'

    values.append(readInt32(file))  # Stars given up
    player['starsGiven'] = values[-1]

    values.append(readInt32(file))  # Damage
    player['damageTaken'] = values[-1]

    values.append(readInt32(file))
    values.append(readInt32(file))  # Attacks used

    values.append(readInt32(file))  # Number of defenses
    player['totalDefenses'] = values[-1]
    numDefenses = values[-1]

    values.append(readInt32(file))  # Gold gained
    values.append(readInt32(file))  # Elixir gained
    values.append(readInt32(file))  # DE gained

    values.append(readInt32(file))  # Gold available
    values.append(readInt32(file))  # Elixir available
    player['goldAndElixir'] = values[-1]

    values.append(readInt32(file))  # DE available
    player['darkElixir'] = values[-1]

    values.append(readInt32(file))  # Offense weight
    player['offenseWeight'] = values[-1]

    values.append(readInt32(file))  # Defense weight
    player['defenseWeight'] = values[-1]

    values.append(readInt32(file))

    values.append(readInt32(file))  # TH level ID
    player['townHall'] = values[-1] + 1

    values.append(readByte(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readByte(file))
    battleDay = values[-1]

    if battleDay:
        values.append(readInt32(file))
        values.append(readInt32(file))

        values.append(readByte(file))
        values.append(readInt32(file) if numDefenses else 0)
        values.append(readInt32(file) if numDefenses else 0) # Best attack replay ID

    values.append(readInt32(file))

    values.append(readInt32(file))  # CC level ID
    values.append(readInt32(file))  # CC capacity
    values.append('"' + readString(file) + '"') # CC request message
    values.append(readInt32(file))  # CC number of troops

    # CC Troops
    sizeI = readInt32(file)
    values.append(sizeI)
    for i in range(0, sizeI):
        values.append(readInt64(file))
        sizeJ = readInt32(file)
        values.append(sizeJ)
        for j in range(0, sizeJ):
            values.append(readInt64(file))
    printList(values)
    return player

def parseAttacks(file, homeClan, enemyClan):
    print(readInt32(fileIn), end = ',', file = fileOut)
    print(readInt32(fileIn), end = ',', file = fileOut)
    numAttacks = readInt32(fileIn)
    print(numAttacks, file = fileOut)

    print(attackLabel, file = fileOut)

    warEvents = []
    for i in range (0, numAttacks):
        warEvents.append(parseAttackEntry(fileIn))

    return warEvents

def parseAttackEntry(file):
    values = []
    event = {}

    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readInt32(file))  # Replay ID

    values.append(readInt32(file))  # Time remaining
    event['timeLeftDisplay'] = secondsToString(values[-1])
    event['timeLeftSeconds'] = values[-1]

    values.append(readInt64(file))  # Attacker clan ID
    attackerClanId = values[-1]

    values.append(readInt64(file))  # Attacker ID
    attackerId = values[-1]

    values.append(readInt64(file))  # Defender clan ID

    values.append(readInt64(file))  # Defender ID
    defenderId = values[-1]

    values.append(readString(file)) # Attacker name
    attackerName = values[-1]

    values.append(readString(file)) # Defender name
    defenderName = values[-1]

    values.append(readInt32(file))  # Stars won
    event['starsWon'] = values[-1]

    values.append(readInt32(file))  # Stars earned
    event['starsEarned'] = values[-1]

    values.append(readInt32(file))  # Damage
    event['damage'] = values[-1]

    values.append(readInt32(file)) # Time taken
    event['timeTaken'] = values[-1] + 1

    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readByte(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    printList(values)

    isHomeAttack = attackerClanId == homeClanId
    event['isHomeAttack'] = isHomeAttack

    homePlayer = searchPlayer(attackerId if isHomeAttack else defenderId, homeClan)
    enemyPlayer = searchPlayer(defenderId if isHomeAttack else attackerId, enemyClan)

    event['homePlayer'] = homePlayer['name']
    event['homePlayerPosition'] = homePlayer['position']
    event['enemyPlayer'] = enemyPlayer['name']
    event['enemyPlayerPosition'] = enemyPlayer['position']

    defenderPosition = searchPlayer(defenderId, enemyClan if isHomeAttack else homeClan)['position']
    attackResult = {'starsWon': event['starsWon'], 'starsEarned': event['starsEarned'], 'damage': event['damage'], 'target': defenderName, 'targetPosition': defenderPosition}

    player = searchPlayer(attackerId, homeClan if isHomeAttack else enemyClan)
    player['attacksUsed'] += 1
    if player['attacksUsed'] == 1:
        player['attack2'] = attackResult
    else:
        player['attack1'] = attackResult

    if event['starsWon'] > 0:
        player['attacksWon'] += 1

    return event

def parsePadding(file):
    values = []
    values.append(readByte(file))
    length = readInt32(file)
    values.append(length)
    for i in range(0, length + 5):
        values.append(readInt32(file))
    values.append(readByte(file))
    printList(values)

def createStats(home, enemy):
    stats = {}
    warSize = home['size']

    stats['attacksUsed'] = 0
    stats['attacksWon'] = 0
    for player in home['roster']:
        stats['attacksUsed'] += player['attacksUsed']
        stats['attacksWon'] += player['attacksWon']
    stats['attacksRemaining'] = warSize * 2 - stats['attacksUsed']
    stats['attacksLost'] = stats['attacksUsed'] - stats['attacksWon']

    stats['3Star'] = 0
    stats['2Star'] = 0
    stats['1Star'] = 0
    destruction = 0
    for player in enemy['roster']:
        destruction += player['damageTaken']
        if player['starsGiven'] == 3:
            stats['3Star'] += 1
        elif player['starsGiven'] == 2:
            stats['2Star'] += 1
        elif player['starsGiven'] == 1:
            stats['1Star'] += 1

    stats['totalStars'] = stats['3Star'] * 3 + stats['2Star'] * 2 + stats['1Star']
    stats['totalDestruction'] = float(destruction) / warSize

    return stats

def createSummary(homeClan, enemyClan, events):
    home = createStats(homeClan, enemyClan)
    enemy = createStats(enemyClan, homeClan)
    if home['totalStars'] > enemy['totalStars']:
        result = "Victory"
    elif home['totalStars'] < enemy['totalStars']:
        result = "Loss"
    else:
        if home['totalDestruction'] > enemy['totalDestruction']:
            result = "Victory"
        elif home['totalDestruction'] < enemy['totalDestruction']:
            result = "Loss"
        else:
            result = "Draw"
    return {'home': home, 'enemy': enemy, 'result': result}

# Labels do not display properly for war packet during preparation day
clanLabel = "ClanID,ClanName,,ClanLevel,WarSize"
memberLabel = "ClanID,MemberID,MemberID,Name,StarsGiven,Damage,,AttacksUsed,TotalDefenses,GoldGained,ElixirGained,DEGained,GoldAvailable,ElixirAvailable,DEAvailable,OffenseWeight,DefenseWeight,,TH,,,,,,,,,,BestAttackReplayID,,CC,CCSize,CCRequestMessage,CCFilled"
attackLabel = ",,ReplayID,TimeLeft,AttackerClanID,AttackerID,DefenderClanID,DefenderID,AttackerName,DefenderName,StarsWon,StarsEarned,Damage"

parser = argparse.ArgumentParser()
parser.add_argument("filepath", help = "decrypted LastClanWarData packet file path")
parser.add_argument("-r", "--raw", action = "store_false", help = "create raw decoded output")
parser.add_argument("-p", "--pretty", action = "store_true", help = "pretty print json")
parser.add_argument("-o", "--output", action = 'store', help = "specify output path, ending with /")
args = parser.parse_args()

fileIn = open(args.filepath, 'rb')
fileOut =  open(args.filepath + "-raw.csv", 'w', encoding = 'utf8')

print(readInt32(fileIn), end = ',', file = fileOut) # Clan war stage
print(readInt32(fileIn), file = fileOut)            # Time left

homeClan = parseClan(fileIn)
parsePadding(fileIn)
enemyClan = parseClan(fileIn)
parsePadding(fileIn)

homeClanId = homeClan['id']
enemyClanId = enemyClan['id']

warEvents = parseAttacks(fileIn, homeClan, enemyClan)
warSummary = createSummary(homeClan, enemyClan, warEvents)

if args.raw:
    fileOut.close()
    os.remove(args.filepath + "-raw.csv") # Should be a better way to do this

# Swap attack order if only attacked once
swapAttackOrder(homeClan)
swapAttackOrder(enemyClan)

m = re.search("[^\\\/]+$", args.filepath)
warData = {'id': int(m.group(0)), 'summary': warSummary, 'home': homeClan, 'enemy': enemyClan, 'events': warEvents}
if args.pretty:
    jsonOutput = json.dumps(warData, indent = 2, ensure_ascii = False)
else:
    jsonOutput = json.dumps(warData, ensure_ascii = False)

if args.output:
    jsonFileOut = open(args.output + m.group(0) + ".json", 'w', encoding = 'utf8')
else:
    jsonFileOut = open(m.group(0) + ".json", 'w', encoding = 'utf8')

print(jsonOutput, file = jsonFileOut)

if not fileIn.read(1):
    print("done parsing")
else:
    print("error: parsing incomplete")<commit_msg>Add best attack<commit_after>import sys
import struct
import os
import argparse
import json
import re

def readInt32(file):
    value = file.read(4)
    if not value:
        return
    return struct.unpack('>i', value)[0]

def readInt64(file):
    value = file.read(8)
    if not value:
        return
    return struct.unpack('>q', value)[0]

def readByte(file):
    return struct.unpack('>B', file.read(1))[0]

def readString(file):
    length = readInt32(file)
    return struct.unpack('%ds' % length, file.read(length))[0].decode('utf-8')

def secondsToString(seconds):
    hours = int(seconds / 3600)
    seconds %= 3600
    minutes = int(seconds / 60)
    seconds %= 60

    str = ""
    if hours > 0:
        str += "%dh " % hours
    if hours == 0 and minutes == 0:
        str += "%ds" % seconds
    else:
        str += "%dm" % minutes

    return str

def searchPlayer(id, clan):
    return [player for player in clan['roster'] if player['id'] == id][0]

def swapAttackOrder(clan):
    for player in clan['roster']:
        if player['attacksUsed'] == 1:
            temp = player['attack1']
            player['attack1'] = player['attack2']
            player['attack2'] = temp

# Print raw output
def printList(values):
    for v in values:
        print(v, end = ',', file = fileOut)
    print("", file = fileOut)

def parseClan(file):
    values = []
    clan = {}

    values.append(readInt64(file))  # Clan ID
    clan['id'] = values[-1]

    values.append(readString(file)) # Clan name
    clan['name'] = values[-1]

    values.append(readInt32(file))  # Clan badge

    values.append(readInt32(file))  # Clan level
    clan['level'] = values[-1]

    values.append(readInt32(file))  # War size
    clan['size'] = values[-1]
    warSize = values[-1]

    print(clanLabel, file = fileOut)
    printList(values)

    print(memberLabel, file = fileOut)

    roster = []
    for i in range(0, warSize):
        player = parseMember(file)
        player['position'] = i + 1
        roster.append(player)

    clan['roster'] = roster
    return clan

def parseMember(file):
    values = [] # raw output
    player = {} # formatted output

    player['attacksUsed'] = 0
    player['attacksWon'] = 0
    player['attack1'] = {'starsWon': 0, 'starsEarned': 0, 'damage': 0, 'target': "", 'targetPosition': 0}
    player['attack2'] = {'starsWon': 0, 'starsEarned': 0, 'damage': 0, 'target': "", 'targetPosition': 0}
    player['enemyBestAttack'] = {'starsWon': 0, 'starsEarned': 0, 'damage': 0, 'target': "", 'targetPosition': 0}

    values.append(readInt64(file))  # Clan ID

    values.append(readInt64(file))  # Member ID
    player['id'] = values[-1]

    values.append(readInt64(file))  # Member ID

    values.append(readString(file)) # Member Name
    player['name'] = values[-1]
    values[-1] = '"' + values[-1] + '"'

    values.append(readInt32(file))  # Stars given up
    player['starsGiven'] = values[-1]

    values.append(readInt32(file))  # Damage
    player['damageTaken'] = values[-1]

    values.append(readInt32(file))
    values.append(readInt32(file))  # Attacks used

    values.append(readInt32(file))  # Number of defenses
    player['totalDefenses'] = values[-1]
    numDefenses = values[-1]

    values.append(readInt32(file))  # Gold gained
    values.append(readInt32(file))  # Elixir gained
    values.append(readInt32(file))  # DE gained

    values.append(readInt32(file))  # Gold available
    values.append(readInt32(file))  # Elixir available
    player['goldAndElixir'] = values[-1]

    values.append(readInt32(file))  # DE available
    player['darkElixir'] = values[-1]

    values.append(readInt32(file))  # Offense weight
    player['offenseWeight'] = values[-1]

    values.append(readInt32(file))  # Defense weight
    player['defenseWeight'] = values[-1]

    values.append(readInt32(file))

    values.append(readInt32(file))  # TH level ID
    player['townHall'] = values[-1] + 1

    values.append(readByte(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readByte(file))
    battleDay = values[-1]

    if battleDay:
        values.append(readInt32(file))
        values.append(readInt32(file))

        values.append(readByte(file))
        values.append(readInt32(file) if numDefenses else 0)
        values.append(readInt32(file) if numDefenses else 0) # Best attack replay ID

    values.append(readInt32(file))

    values.append(readInt32(file))  # CC level ID
    values.append(readInt32(file))  # CC capacity
    values.append('"' + readString(file) + '"') # CC request message
    values.append(readInt32(file))  # CC number of troops

    # CC Troops
    sizeI = readInt32(file)
    values.append(sizeI)
    for i in range(0, sizeI):
        values.append(readInt64(file))
        sizeJ = readInt32(file)
        values.append(sizeJ)
        for j in range(0, sizeJ):
            values.append(readInt64(file))
    printList(values)
    return player

def parseAttacks(file, homeClan, enemyClan):
    print(readInt32(fileIn), end = ',', file = fileOut)
    print(readInt32(fileIn), end = ',', file = fileOut)
    numAttacks = readInt32(fileIn)
    print(numAttacks, file = fileOut)

    print(attackLabel, file = fileOut)

    warEvents = []
    for i in range (0, numAttacks):
        entry = parseAttackEntry(fileIn)
        entry['id'] = numAttacks - i
        warEvents.append(entry)

    return warEvents

def parseAttackEntry(file):
    values = []
    event = {}

    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readInt32(file))  # Replay ID

    values.append(readInt32(file))  # Time remaining
    event['timeLeftDisplay'] = secondsToString(values[-1])
    event['timeLeftSeconds'] = values[-1]

    values.append(readInt64(file))  # Attacker clan ID
    attackerClanId = values[-1]

    values.append(readInt64(file))  # Attacker ID
    attackerId = values[-1]

    values.append(readInt64(file))  # Defender clan ID

    values.append(readInt64(file))  # Defender ID
    defenderId = values[-1]

    values.append(readString(file)) # Attacker name
    attackerName = values[-1]

    values.append(readString(file)) # Defender name
    defenderName = values[-1]

    values.append(readInt32(file))  # Stars won
    event['starsWon'] = values[-1]

    values.append(readInt32(file))  # Stars earned
    event['starsEarned'] = values[-1]

    values.append(readInt32(file))  # Damage
    event['damage'] = values[-1]

    values.append(readInt32(file)) # Time taken
    event['timeTaken'] = values[-1] + 1

    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))

    values.append(readByte(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    values.append(readInt32(file))
    printList(values)

    isHomeAttack = attackerClanId == homeClanId
    event['isHomeAttack'] = isHomeAttack

    homePlayer = searchPlayer(attackerId if isHomeAttack else defenderId, homeClan)
    enemyPlayer = searchPlayer(defenderId if isHomeAttack else attackerId, enemyClan)

    event['homePlayer'] = homePlayer['name']
    event['homePlayerPosition'] = homePlayer['position']
    event['enemyPlayer'] = enemyPlayer['name']
    event['enemyPlayerPosition'] = enemyPlayer['position']

    defender = searchPlayer(defenderId, enemyClan if isHomeAttack else homeClan)
    defenderPosition = defender['position']
    attackResult = {'starsWon': event['starsWon'], 'starsEarned': event['starsEarned'], 'damage': event['damage'], 'target': defenderName, 'targetPosition': defenderPosition}

    player = searchPlayer(attackerId, homeClan if isHomeAttack else enemyClan)
    player['attacksUsed'] += 1
    if player['attacksUsed'] == 1:
        player['attack2'] = attackResult
    else:
        player['attack1'] = attackResult

    if event['starsWon'] > 0:
        player['attacksWon'] += 1

    if (event['starsEarned'] > 0  and event['starsWon'] > defender['enemyBestAttack']['starsWon']) or \
        (event['starsEarned'] == 0 and event['damage'] > defender['enemyBestAttack']['damage']):
        defender['enemyBestAttack'] = {'starsWon': event['starsWon'], 'starsEarned': event['starsEarned'], 'damage': event['damage'], 'target': player['name'], 'targetPosition': player['position']}

    return event

def parsePadding(file):
    values = []
    values.append(readByte(file))
    length = readInt32(file)
    values.append(length)
    for i in range(0, length + 5):
        values.append(readInt32(file))
    values.append(readByte(file))
    printList(values)

def createStats(home, enemy):
    stats = {}
    warSize = home['size']

    stats['attacksUsed'] = 0
    stats['attacksWon'] = 0
    for player in home['roster']:
        stats['attacksUsed'] += player['attacksUsed']
        stats['attacksWon'] += player['attacksWon']
    stats['attacksRemaining'] = warSize * 2 - stats['attacksUsed']
    stats['attacksLost'] = stats['attacksUsed'] - stats['attacksWon']

    stats['3Star'] = 0
    stats['2Star'] = 0
    stats['1Star'] = 0
    destruction = 0
    for player in enemy['roster']:
        destruction += player['damageTaken']
        if player['starsGiven'] == 3:
            stats['3Star'] += 1
        elif player['starsGiven'] == 2:
            stats['2Star'] += 1
        elif player['starsGiven'] == 1:
            stats['1Star'] += 1

    stats['totalStars'] = stats['3Star'] * 3 + stats['2Star'] * 2 + stats['1Star']
    stats['totalDestruction'] = float(destruction) / warSize

    return stats

def createSummary(homeClan, enemyClan, events):
    home = createStats(homeClan, enemyClan)
    enemy = createStats(enemyClan, homeClan)
    if home['totalStars'] > enemy['totalStars']:
        result = "Victory"
    elif home['totalStars'] < enemy['totalStars']:
        result = "Loss"
    else:
        if home['totalDestruction'] > enemy['totalDestruction']:
            result = "Victory"
        elif home['totalDestruction'] < enemy['totalDestruction']:
            result = "Loss"
        else:
            result = "Draw"
    return {'home': home, 'enemy': enemy, 'result': result}

# Labels do not display properly for war packet during preparation day
clanLabel = "ClanID,ClanName,,ClanLevel,WarSize"
memberLabel = "ClanID,MemberID,MemberID,Name,StarsGiven,Damage,,AttacksUsed,TotalDefenses,GoldGained,ElixirGained,DEGained,GoldAvailable,ElixirAvailable,DEAvailable,OffenseWeight,DefenseWeight,,TH,,,,,,,,,,BestAttackReplayID,,CC,CCSize,CCRequestMessage,CCFilled"
attackLabel = ",,ReplayID,TimeLeft,AttackerClanID,AttackerID,DefenderClanID,DefenderID,AttackerName,DefenderName,StarsWon,StarsEarned,Damage"

parser = argparse.ArgumentParser()
parser.add_argument("filepath", help = "decrypted LastClanWarData packet file path")
parser.add_argument("-r", "--raw", action = "store_false", help = "create raw decoded output")
parser.add_argument("-p", "--pretty", action = "store_true", help = "pretty print json")
parser.add_argument("-o", "--output", action = 'store', help = "specify output path, ending with /")
args = parser.parse_args()

fileIn = open(args.filepath, 'rb')
fileOut =  open(args.filepath + "-raw.csv", 'w', encoding = 'utf8')

print(readInt32(fileIn), end = ',', file = fileOut) # Clan war stage
print(readInt32(fileIn), file = fileOut)            # Time left

homeClan = parseClan(fileIn)
parsePadding(fileIn)
enemyClan = parseClan(fileIn)
parsePadding(fileIn)

homeClanId = homeClan['id']
enemyClanId = enemyClan['id']

warEvents = parseAttacks(fileIn, homeClan, enemyClan)
warSummary = createSummary(homeClan, enemyClan, warEvents)

if args.raw:
    fileOut.close()
    os.remove(args.filepath + "-raw.csv") # Should be a better way to do this

# Swap attack order if only attacked once
swapAttackOrder(homeClan)
swapAttackOrder(enemyClan)

m = re.search("[^\\\/]+$", args.filepath)
#warId = int(m.group(0)) if isinstance(m.group(0), int) else 0
warData = {'id': int(m.group(0)), 'summary': warSummary, 'home': homeClan, 'enemy': enemyClan, 'events': warEvents}
if args.pretty:
    jsonOutput = json.dumps(warData, indent = 2, ensure_ascii = False)
else:
    jsonOutput = json.dumps(warData, ensure_ascii = False)

if args.output:
    jsonFileOut = open(args.output + m.group(0) + ".json", 'w', encoding = 'utf8')
else:
    jsonFileOut = open(m.group(0) + ".json", 'w', encoding = 'utf8')

print(jsonOutput, file = jsonFileOut)

if not fileIn.read(1):
    print("done parsing")
else:
    print("error: parsing incomplete")<|endoftext|>


<commit_before># #!/usr/bin/python

import fcntl
import os
import pwd
import grp
import sys
import signal
import resource
import logging
import atexit
from logging import handlers

class Daemonize(object):
    """ Daemonize object
    Object constructor expects three arguments:
    - app: contains the application name which will be sent to syslog.
    - pid: path to the pidfile.
    - action: your custom function which will be executed after daemonization.
    - keep_fds: optional list of fds which should not be closed.
    - privileged_action: action that will be executed before drop privileges if user or
                         group parameter is provided.
    - user: drop privileges to this user if provided.
    - group: drop privileges to this group if provided.
    - verbose: send debug messages to logger if provided.
    """
    def __init__(self, app, pid, action, keep_fds=None, privileged_action=None, user=None, group=None, verbose=False, logger=None):
        self.app = app
        self.pid = pid
        self.action = action
        self.keep_fds = keep_fds or []
        self.privileged_action = privileged_action or (lambda: ())
        self.user = user
        self.group = group
        self.logger = logger
        self.verbose = verbose

    def sigterm(self, signum, frame):
        """ sigterm method
        These actions will be done after SIGTERM.
        """
        self.logger.warn("Caught signal %s. Stopping daemon." % signum)
        os.remove(self.pid)
        sys.exit(0)

    def exit(self):
        """ exit method
        Cleanup pid file at exit.
        """
        self.logger.warn("Stopping daemon.")
        os.remove(self.pid)
        sys.exit(0)

    def start(self):
        """ start method
        Main daemonization process.
        """
        # Fork, creating a new process for the child.
        process_id = os.fork()
        if process_id < 0:
            # Fork error. Exit badly.
            sys.exit(1)
        elif process_id != 0:
            # This is the parent process. Exit.
            sys.exit(0)
        # This is the child process. Continue.

        # Stop listening for signals that the parent process receives.
        # This is done by getting a new process id.
        # setpgrp() is an alternative to setsid().
        # setsid puts the process in a new parent group and detaches its controlling terminal.
        process_id = os.setsid()
        if process_id == -1:
            # Uh oh, there was a problem.
            sys.exit(1)

        # Close all file descriptors, except the ones mentioned in self.keep_fds.
        devnull = "/dev/null"
        if hasattr(os, "devnull"):
            # Python has set os.devnull on this system, use it instead as it might be different
            # than /dev/null.
            devnull = os.devnull

        for fd in range(resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
            if fd not in self.keep_fds:
                try:
                    os.close(fd)
                except OSError:
                    pass

        os.open(devnull, os.O_RDWR)
        os.dup(0)
        os.dup(0)

        if self.logger == None:
            # Initialize logging.
            self.logger = logging.getLogger(self.app)
            self.logger.setLevel(logging.DEBUG)
            # Display log messages only on defined handlers.
            self.logger.propagate = False

            # Initialize syslog.
            # It will work on OS X and Linux. No FreeBSD support, guys, I don't want to import re here
            # to parse your peculiar platform string.
            if sys.platform == "darwin":
                syslog_address = "/var/run/syslog"
            else:
                syslog_address = "/dev/log"
            syslog = handlers.SysLogHandler(syslog_address)
            if self.verbose:
                syslog.setLevel(logging.DEBUG)
            else:
                syslog.setLevel(logging.INFO)
            # Try to mimic to normal syslog messages.
            formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
                                          "%b %e %H:%M:%S")
            syslog.setFormatter(formatter)

            self.logger.addHandler(syslog)

        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
        # octal number which we are typing as 0o27 for Python3 compatibility.
        os.umask(0o27)

        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
        # needs to be deleted results in "directory busy" errors.
        os.chdir("/")

        # Execute privileged action
        priviled_action_result = self.privileged_action()

        # Change gid
        if self.group:
            try:
                gid = grp.getgrnam(self.group).gr_gid
            except KeyError:
                self.logger.error("Group {0} not found".format(self.group))
                sys.exit(1)
            try:
                os.setgid(gid)
            except OSError:
                self.logger.error("Unable to change gid.")
                sys.exit(1)

        # Change uid
        if self.user:
            try:
                uid = pwd.getpwnam(self.user).pw_uid
            except KeyError:
                self.logger.error("User {0} not found.".format(self.user))
                sys.exit(1)
            try:
                os.setuid(uid)
            except OSError:
                self.logger.error("Unable to change uid.")
                sys.exit(1)

        try:
            # Create a lockfile so that only one instance of this daemon is running at any time.
            lockfile = open(self.pid, "w")
        except IOError:
            self.logger.error("Unable to create a pidfile.")
            sys.exit(1)
        try:
            # Try to get an exclusive lock on the file. This will fail if another process has the file
            # locked.
            fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
            # Record the process id to the lockfile. This is standard practice for daemons.
            lockfile.write("%s" % (os.getpid()))
            lockfile.flush()
        except IOError:
            self.logger.error("Unable to lock on the pidfile.")
            os.remove(self.pid)
            sys.exit(1)

        # Set custom action on SIGTERM.
        signal.signal(signal.SIGTERM, self.sigterm)
        atexit.register(self.exit)

        self.logger.warn("Starting daemon.")

        self.action(*priviled_action_result)
<commit_msg>new option for auto_close_fds (default to True)<commit_after># #!/usr/bin/python

import fcntl
import os
import pwd
import grp
import sys
import signal
import resource
import logging
import atexit
from logging import handlers

class Daemonize(object):
    """ Daemonize object
    Object constructor expects three arguments:
    - app: contains the application name which will be sent to syslog.
    - pid: path to the pidfile.
    - action: your custom function which will be executed after daemonization.
    - keep_fds: optional list of fds which should not be closed.
    - privileged_action: action that will be executed before drop privileges if user or
                         group parameter is provided.
    - user: drop privileges to this user if provided.
    - group: drop privileges to this group if provided.
    - verbose: send debug messages to logger if provided.
    """
    def __init__(self, app, pid, action, keep_fds=None, auto_close_fds=True, privileged_action=None, user=None, group=None, verbose=False, logger=None):
        self.app = app
        self.pid = pid
        self.action = action
        self.keep_fds = keep_fds or []
        self.privileged_action = privileged_action or (lambda: ())
        self.user = user
        self.group = group
        self.logger = logger
        self.verbose = verbose
        self.auto_close_fds = auto_close_fds

    def sigterm(self, signum, frame):
        """ sigterm method
        These actions will be done after SIGTERM.
        """
        self.logger.warn("Caught signal %s. Stopping daemon." % signum)
        os.remove(self.pid)
        sys.exit(0)

    def exit(self):
        """ exit method
        Cleanup pid file at exit.
        """
        self.logger.warn("Stopping daemon.")
        os.remove(self.pid)
        sys.exit(0)

    def start(self):
        """ start method
        Main daemonization process.
        """
        # Fork, creating a new process for the child.
        process_id = os.fork()
        if process_id < 0:
            # Fork error. Exit badly.
            sys.exit(1)
        elif process_id != 0:
            # This is the parent process. Exit.
            sys.exit(0)
        # This is the child process. Continue.

        # Stop listening for signals that the parent process receives.
        # This is done by getting a new process id.
        # setpgrp() is an alternative to setsid().
        # setsid puts the process in a new parent group and detaches its controlling terminal.
        process_id = os.setsid()
        if process_id == -1:
            # Uh oh, there was a problem.
            sys.exit(1)

        # Close all file descriptors, except the ones mentioned in self.keep_fds.
        devnull = "/dev/null"
        if hasattr(os, "devnull"):
            # Python has set os.devnull on this system, use it instead as it might be different
            # than /dev/null.
            devnull = os.devnull

        if self.auto_close_fds:
            for fd in range(resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
                if fd not in self.keep_fds:
                    try:
                        os.close(fd)
                    except OSError:
                        pass
            os.open(devnull, os.O_RDWR)
            os.dup(0)
            os.dup(0)

        if self.logger == None:
            # Initialize logging.
            self.logger = logging.getLogger(self.app)
            self.logger.setLevel(logging.DEBUG)
            # Display log messages only on defined handlers.
            self.logger.propagate = False

            # Initialize syslog.
            # It will work on OS X and Linux. No FreeBSD support, guys, I don't want to import re here
            # to parse your peculiar platform string.
            if sys.platform == "darwin":
                syslog_address = "/var/run/syslog"
            else:
                syslog_address = "/dev/log"
            syslog = handlers.SysLogHandler(syslog_address)
            if self.verbose:
                syslog.setLevel(logging.DEBUG)
            else:
                syslog.setLevel(logging.INFO)
            # Try to mimic to normal syslog messages.
            formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
                                          "%b %e %H:%M:%S")
            syslog.setFormatter(formatter)

            self.logger.addHandler(syslog)

        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
        # octal number which we are typing as 0o27 for Python3 compatibility.
        os.umask(0o27)

        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
        # needs to be deleted results in "directory busy" errors.
        os.chdir("/")

        # Execute privileged action
        priviled_action_result = self.privileged_action()

        # Change gid
        if self.group:
            try:
                gid = grp.getgrnam(self.group).gr_gid
            except KeyError:
                self.logger.error("Group {0} not found".format(self.group))
                sys.exit(1)
            try:
                os.setgid(gid)
            except OSError:
                self.logger.error("Unable to change gid.")
                sys.exit(1)

        # Change uid
        if self.user:
            try:
                uid = pwd.getpwnam(self.user).pw_uid
            except KeyError:
                self.logger.error("User {0} not found.".format(self.user))
                sys.exit(1)
            try:
                os.setuid(uid)
            except OSError:
                self.logger.error("Unable to change uid.")
                sys.exit(1)

        try:
            # Create a lockfile so that only one instance of this daemon is running at any time.
            lockfile = open(self.pid, "w")
        except IOError:
            self.logger.error("Unable to create a pidfile.")
            sys.exit(1)
        try:
            # Try to get an exclusive lock on the file. This will fail if another process has the file
            # locked.
            fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
            # Record the process id to the lockfile. This is standard practice for daemons.
            lockfile.write("%s" % (os.getpid()))
            lockfile.flush()
        except IOError:
            self.logger.error("Unable to lock on the pidfile.")
            os.remove(self.pid)
            sys.exit(1)

        # Set custom action on SIGTERM.
        signal.signal(signal.SIGTERM, self.sigterm)
        atexit.register(self.exit)

        self.logger.warn("Starting daemon.")

        self.action(*priviled_action_result)
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
@brief      test log(time=32s)
"""

import sys
import os
import unittest
import shutil

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import jyquickhelper as skip___
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "jyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import jyquickhelper as skip___

try:
    import pyensae as skip__
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyensae",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyensae as skip__

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder
from pyquickhelper.pycode import fix_tkinter_issues_virtualenv
from pyquickhelper.ipythonhelper import execute_notebook_list_finalize_ut
from src.ensae_projects.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
import src.ensae_projects

class TestNotebookCheatSheet(unittest.TestCase):

    def test_notebook_cheatsheet(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        fix_tkinter_issues_virtualenv()
        temp = get_temp_folder(__file__, "temp_cheat_sheet")
        keepnote = ls_notebooks("cheat_sheets")
        self.assertTrue(len(keepnote) > 0)
        keepnote = [_ for _ in keepnote if "chsh_files" not in _]

        if len(keepnote) > 0:
            fold = os.path.dirname(keepnote[0])
            copy = [os.path.join(fold, "geo_data.zip")]
            for c in copy:
                shutil.copy(c, temp)

        res = execute_notebooks(temp, keepnote,
                                lambda i, n: "deviner" not in n and "html" not in n,
                                fLOG=fLOG,
                                clean_function=clean_function_notebook)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=src.ensae_projects)

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix unit test<commit_after>#-*- coding: utf-8 -*-
"""
@brief      test log(time=32s)
"""

import sys
import os
import unittest
import shutil

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import jyquickhelper as skip___
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "jyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import jyquickhelper as skip___

try:
    import pyensae as skip__
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyensae",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyensae as skip__

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder
from pyquickhelper.pycode import fix_tkinter_issues_virtualenv
from pyquickhelper.ipythonhelper import execute_notebook_list_finalize_ut
from src.ensae_projects.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
import src.ensae_projects

class TestNotebookCheatSheet(unittest.TestCase):

    def test_notebook_cheatsheet(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        fix_tkinter_issues_virtualenv()
        temp = get_temp_folder(__file__, "temp_cheat_sheet")
        keepnote = ls_notebooks("cheat_sheets")
        self.assertTrue(len(keepnote) > 0)
        keepnote = [_ for _ in keepnote if "chsh_files" not in _]

        if len(keepnote) > 0:
            fold = os.path.dirname(keepnote[0])
            copy = [os.path.join(fold, "geo_data.zip"),
                    os.path.join(fold, "images1.jpg")]
            for c in copy:
                shutil.copy(c, temp)

        res = execute_notebooks(temp, keepnote,
                                lambda i, n: "deviner" not in n and "html" not in n,
                                fLOG=fLOG,
                                clean_function=clean_function_notebook)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=src.ensae_projects)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function)

from qtpy.QtWidgets import QDialog
from addie.utilities import load_ui
from qtpy import QtGui

from addie.processing.mantid.master_table.table_row_handler import TableRowHandler
from addie.utilities.math_tools import is_number

class DimensionsSetter(QDialog):

    shape_selected = 'cylindrical'

    def __init__(self, parent=None, key=None, data_type='sample'):
        self.parent = parent
        self.key = key
        self.data_type =  data_type

        QDialog.__init__(self, parent=parent)
        self.ui = load_ui('dimensions_setter.ui', baseinstance=self)

        self.group_widgets()
        self.init_widgets_layout()
        self.init_widgets_content()

        if parent.geometry_ui_position:
            self.move(parent.geometry_ui_position)

        self.check_save_button()

    def group_widgets(self):
        self.group = {'radius': [self.ui.radius_label,
                                 self.ui.radius_value,
                                 self.ui.radius_units],
                      'radius2': [self.ui.radius2_label,
                                  self.ui.radius2_value,
                                  self.ui.radius2_units],
                      'height': [self.ui.height_label,
                                 self.ui.height_value,
                                 self.ui.height_units]}

    def __get_label_value(self, geometry_type):
        '''helper function to retrieve value of labels from master table.

        :argument:
        geometry_type being 'radius', 'radius2' or 'height'
        '''
        return str(self.parent.master_table_list_ui[self.key][self.data_type]['geometry'][geometry_type]['value'].text())

    def __set_label_value(self, geometry_type, value):
        '''helper function to set value of label in master table.

        :argument:
        geometry_type being 'radius', 'radius2' or 'height'
        value: value to set
        '''
        self.parent.master_table_list_ui[self.key][self.data_type]['geometry'][geometry_type]['value'].setText(value)

    def init_widgets_content(self):
        '''populate the widgets using the value from the master table'''

        height = 'N/A'
        radius2 = 'N/A'

        if self.shape_selected.lower() == 'cylindrical':
            radius = self.__get_label_value('radius')
            height = self.__get_label_value('height')
        elif self.shape_selected.lower() == 'spherical':
            radius = self.__get_label_value('radius')
        else:
            radius = self.__get_label_value('radius')
            radius2 = self.__get_label_value('radius2')
            height = self.__get_label_value('height')

        self.ui.radius_value.setText(radius)
        self.ui.radius2_value.setText(radius2)
        self.ui.height_value.setText(height)

    def init_widgets_layout(self):
        '''using the shape defined for this row, will display the right widgets and will populate
        them with the right values'''

        # which shape are we working on
        table_row_ui = self.parent.master_table_list_ui[self.key][self.data_type]
        shape_ui = table_row_ui['shape']
        self.shape_selected = shape_ui.currentText()

        # hide/show widgets according to shape selected
        if self.shape_selected.lower() == 'cylindrical':
            # change label of first label
            self.ui.radius_label.setText("Radius")
            # hide radius 2 widgets
            for _widget in self.group['radius2']:
                _widget.setVisible(False)
            # display right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/cylindrical_reference.png"))
            self.ui.preview.setScaledContents(True)

        elif self.shape_selected.lower() == 'spherical':
            # change label of first label
            self.ui.radius_label.setText("Radius")
            # hide radius widgets
            for _widget in self.group['radius2']:
                _widget.setVisible(False)
            # hide radius 2 widgets
            for _widget in self.group['height']:
                _widget.setVisible(False)
            # display the right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/spherical_reference.png"))
            self.ui.preview.setScaledContents(True)

        elif self.shape_selected.lower() == 'hollow cylinder':
            # display the right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/hollow_cylindrical_reference.png"))
            self.ui.preview.setScaledContents(True)

        # display value of radius1,2,height for this row

        return

    def value_changed(self, text):
        self.check_save_button()

    def check_save_button(self):
        save_button_status = False

        radius = str(self.ui.radius_value.text())
        radius2 = str(self.ui.radius2_value.text())
        height = str(self.ui.height_value.text())

        if self.shape_selected.lower() == 'cylindrical':
            if is_number(radius) and is_number(height):
                save_button_status = True
        elif self.shape_selected.lower() == 'spherical':
            if is_number(radius):
                save_button_status = True
        else:
            if is_number(radius) and is_number(radius2) and is_number(height):
                save_button_status = True

        self.ui.ok.setEnabled(save_button_status)

    def accept(self):

        radius = str(self.ui.radius_value.text())
        radius2 = 'N/A'
        height = 'N/A'

        if self.shape_selected.lower() == 'cylindrical':
            height = str(self.ui.height_value.text())
        elif self.shape_selected.lower() == 'spherical':
            pass
        else:
            radius2 = str(self.ui.radius2_value.text())
            height = str(self.ui.height_value.text())

        self.__set_label_value('radius', radius)
        self.__set_label_value('radius2', radius2)
        self.__set_label_value('height', height)

        o_table = TableRowHandler(parent=self.parent)
        o_table.transfer_widget_states(from_key=self.key, data_type=self.data_type)

        self.close()

    def closeEvent(self, c):
        self.parent.geometry_ui_position = self.pos()
<commit_msg>Bug fix for geometry save button<commit_after>from __future__ import (absolute_import, division, print_function)

from qtpy.QtWidgets import QDialog
from addie.utilities import load_ui
from qtpy import QtGui

from addie.processing.mantid.master_table.table_row_handler import TableRowHandler
from addie.utilities.math_tools import is_number

class DimensionsSetter(QDialog):

    shape_selected = 'cylindrical'

    def __init__(self, parent=None, key=None, data_type='sample'):
        self.parent = parent
        self.key = key
        self.data_type =  data_type

        QDialog.__init__(self, parent=parent)
        self.ui = load_ui('dimensions_setter.ui', baseinstance=self)

        self.group_widgets()
        self.init_widgets_layout()
        self.init_widgets_content()

        if parent.geometry_ui_position:
            self.move(parent.geometry_ui_position)

        self.check_save_button()

    def group_widgets(self):
        self.group = {'radius': [self.ui.radius_label,
                                 self.ui.radius_value,
                                 self.ui.radius_units],
                      'radius2': [self.ui.radius2_label,
                                  self.ui.radius2_value,
                                  self.ui.radius2_units],
                      'height': [self.ui.height_label,
                                 self.ui.height_value,
                                 self.ui.height_units]}

    def __get_label_value(self, geometry_type):
        '''helper function to retrieve value of labels from master table.

        :argument:
        geometry_type being 'radius', 'radius2' or 'height'
        '''
        return str(self.parent.master_table_list_ui[self.key][self.data_type]['geometry'][geometry_type]['value'].text())

    def __set_label_value(self, geometry_type, value):
        '''helper function to set value of label in master table.

        :argument:
        geometry_type being 'radius', 'radius2' or 'height'
        value: value to set
        '''
        self.parent.master_table_list_ui[self.key][self.data_type]['geometry'][geometry_type]['value'].setText(value)

    def init_widgets_content(self):
        '''populate the widgets using the value from the master table'''

        height = 'N/A'
        radius2 = 'N/A'

        if self.shape_selected.lower() == 'cylindrical':
            radius = self.__get_label_value('radius')
            height = self.__get_label_value('height')
        elif self.shape_selected.lower() == 'spherical':
            radius = self.__get_label_value('radius')
        else:
            radius = self.__get_label_value('radius')
            radius2 = self.__get_label_value('radius2')
            height = self.__get_label_value('height')

        self.ui.radius_value.setText(radius)
        self.ui.radius2_value.setText(radius2)
        self.ui.height_value.setText(height)

    def init_widgets_layout(self):
        '''using the shape defined for this row, will display the right widgets and will populate
        them with the right values'''

        # which shape are we working on
        table_row_ui = self.parent.master_table_list_ui[self.key][self.data_type]
        shape_ui = table_row_ui['shape']
        self.shape_selected = shape_ui.currentText()

        # hide/show widgets according to shape selected
        if self.shape_selected.lower() == 'cylindrical':
            # change label of first label
            self.ui.radius_label.setText("Radius")
            # hide radius 2 widgets
            for _widget in self.group['radius2']:
                _widget.setVisible(False)
            # display right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/cylindrical_reference.png"))
            self.ui.preview.setScaledContents(True)

        elif self.shape_selected.lower() == 'spherical':
            # change label of first label
            self.ui.radius_label.setText("Radius")
            # hide radius widgets
            for _widget in self.group['radius2']:
                _widget.setVisible(False)
            # hide radius 2 widgets
            for _widget in self.group['height']:
                _widget.setVisible(False)
            # display the right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/spherical_reference.png"))
            self.ui.preview.setScaledContents(True)

        elif self.shape_selected.lower() == 'hollow cylinder':
            # display the right image label
            self.ui.preview.setPixmap(QtGui.QPixmap(":/preview/hollow_cylindrical_reference.png"))
            self.ui.preview.setScaledContents(True)

        # display value of radius1,2,height for this row

        return

    def value_changed(self, text):
        self.check_save_button()

    def check_save_button(self):
        save_button_status = False

        radius = str(self.ui.radius_value.text())
        radius2 = str(self.ui.radius2_value.text())
        height = str(self.ui.height_value.text())

        if self.shape_selected.lower() == 'cylindrical':
            if is_number(radius) and is_number(height):
                save_button_status = True
        elif self.shape_selected.lower() == 'spherical':
            if is_number(radius):
                save_button_status = True
        else:
            if is_number(radius) and is_number(radius2) and is_number(height):
                save_button_status = True

        self.ui.ok.setEnabled(save_button_status)

    def accept(self):

        radius = str(self.ui.radius_value.text())
        radius2 = 'N/A'
        height = 'N/A'

        if self.shape_selected.lower() == 'cylindrical':
            height = str(self.ui.height_value.text())
        elif self.shape_selected.lower() == 'spherical':
            pass
        else:
            radius2 = str(self.ui.radius2_value.text())
            height = str(self.ui.height_value.text())

        self.__set_label_value('radius', radius)
        self.__set_label_value('radius2', radius2)
        self.__set_label_value('height', height)

        o_table = TableRowHandler(main_window=self.parent)
        o_table.transfer_widget_states(from_key=self.key, data_type=self.data_type)

        self.close()

    def closeEvent(self, c):
        self.parent.geometry_ui_position = self.pos()
<|endoftext|>


<commit_before># Generated by Django 3.1.8 on 2021-05-12 11:55

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('notifications', '0008_initials'),
    ]

    operations = [
        migrations.AlterField(
            model_name='subscription',
            name='activity_type',
            field=models.CharField(choices=[('waiting-payment', 'Paiement en attente'), ('group-invitation', 'Invitation à un groupe'), ('new-member', 'Nouveau membre dans le groupe'), ('group-membership-limit-reminder', 'Les membres du groupes sont de plus en plus nombreux'), ('new-message', 'Nouveau message dans un de vos groupes'), ('new-comment', 'Nouveau commentaire dans une de vos discussions'), ('waiting-location-group', 'Préciser la localisation du groupe'), ('group-coorganization-invite', 'Invitation à coorganiser un groupe reçue'), ('waiting-location-event', "Préciser la localisation d'un événement"), ('group-coorganization-accepted', 'Invitation à coorganiser un groupe acceptée'), ('group-info-update', 'Mise à jour des informations du groupe'), ('accepted-invitation-member', 'Invitation à rejoindre un groupe acceptée'), ('new-attendee', 'Un nouveau participant à votre événement'), ('event-update', "Mise à jour d'un événement"), ('new-event-mygroups', 'Votre groupe organise un événement'), ('new-report', "Nouveau compte-rendu d'événement"), ('new-event-aroundme', 'Nouvel événement près de chez moi'), ('cancelled-event', 'Événement annulé'), ('referral-accepted', 'Personne parrainée'), ('announcement', 'Associée à une annonce'), ('transferred-group-member', "Un membre d'un groupe a été transferé vers un autre groupe"), ('new-members-through-transfer', 'De nouveaux membres ont été transferés vers le groupe'), ('group-creation-confirmation', 'Groupe créé'), ('event-suggestion', 'Événement suggéré')], max_length=50, verbose_name='Type'),
        ),
    ]
<commit_msg>reformat migration<commit_after># Generated by Django 3.1.8 on 2021-05-12 11:55

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [("notifications", "0008_initials")]

    operations = [
        migrations.AlterField(
            model_name="subscription",
            name="activity_type",
            field=models.CharField(
                choices=[
                    ("waiting-payment", "Paiement en attente"),
                    ("group-invitation", "Invitation à un groupe"),
                    ("new-member", "Nouveau membre dans le groupe"),
                    (
                        "group-membership-limit-reminder",
                        "Les membres du groupes sont de plus en plus nombreux",
                    ),
                    ("new-message", "Nouveau message dans un de vos groupes"),
                    ("new-comment", "Nouveau commentaire dans une de vos discussions"),
                    ("waiting-location-group", "Préciser la localisation du groupe"),
                    (
                        "group-coorganization-invite",
                        "Invitation à coorganiser un groupe reçue",
                    ),
                    (
                        "waiting-location-event",
                        "Préciser la localisation d'un événement",
                    ),
                    (
                        "group-coorganization-accepted",
                        "Invitation à coorganiser un groupe acceptée",
                    ),
                    ("group-info-update", "Mise à jour des informations du groupe"),
                    (
                        "accepted-invitation-member",
                        "Invitation à rejoindre un groupe acceptée",
                    ),
                    ("new-attendee", "Un nouveau participant à votre événement"),
                    ("event-update", "Mise à jour d'un événement"),
                    ("new-event-mygroups", "Votre groupe organise un événement"),
                    ("new-report", "Nouveau compte-rendu d'événement"),
                    ("new-event-aroundme", "Nouvel événement près de chez moi"),
                    ("cancelled-event", "Événement annulé"),
                    ("referral-accepted", "Personne parrainée"),
                    ("announcement", "Associée à une annonce"),
                    (
                        "transferred-group-member",
                        "Un membre d'un groupe a été transferé vers un autre groupe",
                    ),
                    (
                        "new-members-through-transfer",
                        "De nouveaux membres ont été transferés vers le groupe",
                    ),
                    ("group-creation-confirmation", "Groupe créé"),
                    ("event-suggestion", "Événement suggéré"),
                ],
                max_length=50,
                verbose_name="Type",
            ),
        )
    ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Copyright © 2016, Continuum Analytics, Inc. All rights reserved.
#
# The full license is in the file LICENSE.txt, distributed with this software.
# ----------------------------------------------------------------------------
from __future__ import absolute_import, print_function

import pytest

from anaconda_project.commands.variable_commands import main
from anaconda_project.internal.test.tmpfile_utils import with_directory_contents
from anaconda_project.project_file import DEFAULT_PROJECT_FILENAME

class Args(object):
    def __init__(self, action, vars_to_set=None, vars_to_unset=None, project='.'):
        self.project = project
        self.action = action
        self.environment = 'default'
        self.vars_to_set = vars_to_set
        self.vars_to_unset = vars_to_unset

def test_set_variable_command(monkeypatch):

    params = []

    def mock_add_variables(project, _vars):
        params.append(_vars)
        return True

    monkeypatch.setattr('anaconda_project.project_ops.add_variables', mock_add_variables)

    args = Args('set', vars_to_set=['foo=bar', 'baz=qux', 'has_two_equals=foo=bar'])
    res = main(args)
    assert res == 0
    assert [ ('foo', 'bar'), ('baz', 'qux'), ('has_two_equals', 'foo=bar') ] == params[0]

def test_set_variable_project_problem(capsys):
    def check_problem(dirname):
        args = Args('set', vars_to_set=['foo=bar', 'baz=qux'], project=dirname)
        res = main(args)
        assert res == 1

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """
runtime:
  42
"""}, check_problem)

    out, err = capsys.readouterr()
    assert out == ''
    expected_err = ('Unable to load project:\n  '
                    'runtime section contains wrong value type 42, should be dict or list of requirements\n')
    assert err == expected_err

def test_set_variable_command_bad(monkeypatch, capsys):

    params = []

    def mock_add_variables(project, _vars):
        params.append(_vars)
        return True

    monkeypatch.setattr('anaconda_project.project_ops.add_variables', mock_add_variables)

    args = Args('set', vars_to_set=['foo=bar', 'baz'])
    res = main(args)
    assert res == 1
    out, err = capsys.readouterr()
    assert "Error: {} doesn't define a name=value pair".format('baz') in out

    assert len(params) == 0

def test_unset_variable_command(monkeypatch):

    args = Args('unset', vars_to_unset=['foo=bar', 'baz=qux'])
    with pytest.raises(NotImplementedError):
        main(args)
<commit_msg>test_variable_commands.py reformatting<commit_after># -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Copyright © 2016, Continuum Analytics, Inc. All rights reserved.
#
# The full license is in the file LICENSE.txt, distributed with this software.
# ----------------------------------------------------------------------------
from __future__ import absolute_import, print_function

import pytest

from anaconda_project.commands.variable_commands import main
from anaconda_project.internal.test.tmpfile_utils import with_directory_contents
from anaconda_project.project_file import DEFAULT_PROJECT_FILENAME

class Args(object):
    def __init__(self, action, vars_to_set=None, vars_to_unset=None, project='.'):
        self.project = project
        self.action = action
        self.environment = 'default'
        self.vars_to_set = vars_to_set
        self.vars_to_unset = vars_to_unset

def test_set_variable_command(monkeypatch):

    params = []

    def mock_add_variables(project, _vars):
        params.append(_vars)
        return True

    monkeypatch.setattr('anaconda_project.project_ops.add_variables', mock_add_variables)

    args = Args('set', vars_to_set=['foo=bar', 'baz=qux', 'has_two_equals=foo=bar'])
    res = main(args)
    assert res == 0
    assert [('foo', 'bar'), ('baz', 'qux'), ('has_two_equals', 'foo=bar')] == params[0]

def test_set_variable_project_problem(capsys):
    def check_problem(dirname):
        args = Args('set', vars_to_set=['foo=bar', 'baz=qux'], project=dirname)
        res = main(args)
        assert res == 1

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """
runtime:
  42
"""}, check_problem)

    out, err = capsys.readouterr()
    assert out == ''
    expected_err = ('Unable to load project:\n  '
                    'runtime section contains wrong value type 42, should be dict or list of requirements\n')
    assert err == expected_err

def test_set_variable_command_bad(monkeypatch, capsys):

    params = []

    def mock_add_variables(project, _vars):
        params.append(_vars)
        return True

    monkeypatch.setattr('anaconda_project.project_ops.add_variables', mock_add_variables)

    args = Args('set', vars_to_set=['foo=bar', 'baz'])
    res = main(args)
    assert res == 1
    out, err = capsys.readouterr()
    assert "Error: {} doesn't define a name=value pair".format('baz') in out

    assert len(params) == 0

def test_unset_variable_command(monkeypatch):

    args = Args('unset', vars_to_unset=['foo=bar', 'baz=qux'])
    with pytest.raises(NotImplementedError):
        main(args)
<|endoftext|>


<commit_before>import matplotlib, os, errno
# IF WE ARE ON SERVER WITH NO DISPLAY, then:
#print matplotlib.get_backend()
if not('DISPLAY' in os.environ):
    matplotlib.use("Agg")

import matplotlib.pyplot as plt

import numpy as np

'''
Example calls:
hi = model.fit(...)

saveHistory(hi.history, 'tmp_saved_history.npy')
visualize_history(loadHistory('tmp_saved_history.npy'))

'''

def visualize_history(hi, show=True, save=False, save_path='', show_also='', custom_title=None):

    # list all data in history
    print(hi.keys())
    # summarize history for loss
    plt.plot(hi['loss'])
    plt.plot(hi['val_loss'])

    if show_also <> '':
        plt.plot(hi[show_also], linestyle='dotted')
        plt.plot(hi['val_'+show_also], linestyle='dotted')

    if custom_title is None:
        plt.title('model loss')
    else:
        plt.title(custom_title)

    plt.ylabel('loss')
    plt.xlabel('epoch')

    if show_also == '':
        plt.legend(['train', 'test'], loc='upper left')
    else:
        plt.legend(['train', 'test', 'train-'+show_also, 'test-'+show_also], loc='upper left')

    if save:
        filename = save_path #+'loss.png'
        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError as exc:
                if exc.errno != errno.EEXIST:
                    raise

        plt.savefig(filename)
        print "Saved image to "+filename
    if show:
        plt.show()

    plt.clf()
    return plt

def visualize_histories(histories, names, plotvalues='loss', show=True, save=False, save_path='', custom_title=None, just_val=False):

    import matplotlib.pyplot as plt

    '''
    Visualize multiple histories.

    Example usage:
        h1 = loadHistory('history1.npy')
        h2 = loadHistory('history2.npy')
        visualize_histories([h1, h2], ['history1', 'history2'])
    '''
    if custom_title is None:
        custom_title = 'model ' + plotvalues
    if just_val:
        custom_title = custom_title + ' (just validation results)'

    i = 0
    leg = []
    for hi in histories:
        n = names[i]
        # list all data in history
        print(hi.keys())
        # summarize history for loss
        if not just_val:
            plt.plot(hi[plotvalues])
        plt.plot(hi['val_'+plotvalues])
        plt.title(custom_title)
        plt.ylabel('loss')
        plt.xlabel('epoch')
        if not just_val:
            leg.append(n + '')
        leg.append(n + '_val')
        i += 1
    #plt.legend(leg, loc='lower left')
    plt.legend(leg, loc='best')
    if save:
        plt.savefig(save_path) #+plotvalues+'.png')
    if show:
        plt.show()

    plt.clf()
    return plt

def saveHistory(history_dict, filename):
    if not os.path.exists(os.path.dirname(filename)):
        try:
            os.makedirs(os.path.dirname(filename))
        except OSError as exc: # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise

    to_be_saved = data = {'S': history_dict}
    np.save(open(filename, 'w'), to_be_saved)

def loadHistory(filename):
    loaded = np.load(open(filename))
    return loaded[()]['S']

#visualize_history(loadHistory('1100downloaded_vII/tmp_saved_history.npy'))

#h1 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/1_SimpleCNN_3conv_2fc/results/history.npy')
#h2 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/2_VGG16_no_finetuning/(runs)/now2/results/history.npy')
#h3 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/2_VGG16_no_finetuning/results/history_finetune.npy')
#visualize_histories([h1, h2, h3], ['simple', 'vgg', 'finetune'],save=True)

<commit_msg>PDF output too<commit_after>import matplotlib, os, errno
# IF WE ARE ON SERVER WITH NO DISPLAY, then:
#print matplotlib.get_backend()
if not('DISPLAY' in os.environ):
    matplotlib.use("Agg")

import matplotlib.pyplot as plt

import numpy as np

'''
Example calls:
hi = model.fit(...)

saveHistory(hi.history, 'tmp_saved_history.npy')
visualize_history(loadHistory('tmp_saved_history.npy'))

'''

def visualize_history(hi, show=True, save=False, save_path='', show_also='', custom_title=None):

    # list all data in history
    print(hi.keys())
    # summarize history for loss
    plt.plot(hi['loss'])
    plt.plot(hi['val_loss'])

    if show_also <> '':
        plt.plot(hi[show_also], linestyle='dotted')
        plt.plot(hi['val_'+show_also], linestyle='dotted')

    if custom_title is None:
        plt.title('model loss')
    else:
        plt.title(custom_title)

    plt.ylabel('loss')
    plt.xlabel('epoch')

    if show_also == '':
        plt.legend(['train', 'test'], loc='upper left')
    else:
        plt.legend(['train', 'test', 'train-'+show_also, 'test-'+show_also], loc='upper left')

    if save:
        filename = save_path #+'loss.png'
        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError as exc:
                if exc.errno != errno.EEXIST:
                    raise

        plt.savefig(filename)
        plt.savefig(filename+'.pdf', format='pdf')

        print "Saved image to "+filename
    if show:
        plt.show()

    plt.clf()
    return plt

def visualize_histories(histories, names, plotvalues='loss', show=True, save=False, save_path='', custom_title=None, just_val=False):

    import matplotlib.pyplot as plt

    '''
    Visualize multiple histories.

    Example usage:
        h1 = loadHistory('history1.npy')
        h2 = loadHistory('history2.npy')
        visualize_histories([h1, h2], ['history1', 'history2'])
    '''
    if custom_title is None:
        custom_title = 'model ' + plotvalues
    if just_val:
        custom_title = custom_title + ' (just validation results)'

    i = 0
    leg = []
    for hi in histories:
        n = names[i]
        # list all data in history
        print(hi.keys())
        # summarize history for loss
        if not just_val:
            plt.plot(hi[plotvalues])
        plt.plot(hi['val_'+plotvalues])
        plt.title(custom_title)
        plt.ylabel('loss')
        plt.xlabel('epoch')
        if not just_val:
            leg.append(n + '')
        leg.append(n + '_val')
        i += 1
    #plt.legend(leg, loc='lower left')
    plt.legend(leg, loc='best')
    if save:
        plt.savefig(save_path) #+plotvalues+'.png')
        plt.savefig(save_path+'.pdf', format='pdf')

    if show:
        plt.show()

    plt.clf()
    return plt

def saveHistory(history_dict, filename):
    if not os.path.exists(os.path.dirname(filename)):
        try:
            os.makedirs(os.path.dirname(filename))
        except OSError as exc: # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise

    to_be_saved = data = {'S': history_dict}
    np.save(open(filename, 'w'), to_be_saved)

def loadHistory(filename):
    loaded = np.load(open(filename))
    return loaded[()]['S']

#visualize_history(loadHistory('1100downloaded_vII/tmp_saved_history.npy'))

#h1 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/1_SimpleCNN_3conv_2fc/results/history.npy')
#h2 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/2_VGG16_no_finetuning/(runs)/now2/results/history.npy')
#h3 = loadHistory('/home/ekmek/TEMP_SPACE/MGR-Project-Code/Downloader/ModelImplementations/2_VGG16_no_finetuning/results/history_finetune.npy')
#visualize_histories([h1, h2, h3], ['simple', 'vgg', 'finetune'],save=True)

<|endoftext|>