<commit_before>import pkg_resources
import csv
import re
from nala.learning.evaluators import is_overlapping
from nala.structures.data import Annotation

class PostProcessing:
    def __init__(self):
        self.at_least_one_letter_n_number_letter_n_number = re.compile('(?=.*[A-Za-z])(?=.*[0-9])[A-Za-z0-9]+')
        self.short = re.compile('^[A-Z][0-9][A-Z]$')

    def process(self, dataset):
        existing_predictions = []

        for doc_id, doc in dataset.documents.items():
            for part_id, part in doc.parts.items():
                self.__fix_issues(part)

                for index, ann in enumerate(part.predicted_annotations):
                    existing_predictions.append((ann.offset, ann.offset + len(ann.text), part_id, ann.class_id, doc_id, index))

        regex_patterns = construct_regex_patterns_from_predictions(dataset)

        for doc_id, doc in dataset.documents.items():
            for part_id, part in doc.parts.items():
                for regex in regex_patterns:
                    for match in regex.finditer(part.text):
                        offset = (match.start(), match.end(), part_id, 'e_2', doc_id)
                        matched_text = part.text[match.start():match.end()]

                        # TODO Refactor into regex instead of check
                        try:
                            space_before = part.text[match.start() - 1] == ' '
                        except IndexError:
                            space_before = True
                        try:
                            space_after = part.text[match.end()] == ' '
                        except IndexError:
                            space_after = True

                        if not is_overlapping(offset, existing_predictions):
                            if not self.short.search(matched_text) and space_before and space_after \
                                    and self.at_least_one_letter_n_number_letter_n_number.search(matched_text):
                                existing_predictions.append(offset)
                                part.predicted_annotations.append(Annotation('e_2', match.start(), matched_text))
                        else:
                            # TODO Refactor to return an object
                            # our custom part (needs to be optimized) adds 1% to the f_measure (eg. 87 to 88)
                            for offset_b in existing_predictions:
                                # if there is a partial overlap with a regex match
                                if offset[2:5] == offset_b[2:5] and offset[0] <= offset_b[1] and offset[1] >= offset_b[0]:
                                    # but that overlap does not contain spaces and cover text span bigger in length
                                    if (offset[1] - offset[0]) > (offset_b[1]-offset_b[0]) and ' ' not in matched_text:
                                        # and additionally it's between two spaces
                                        if re.search(' +{} +'.format(regex.pattern), part.text):
                                            # replace the existing one by the one found one since it is probably better
                                            dataset.documents[offset_b[-2]].parts[offset_b[-4]].predicted_annotations[offset_b[-1]] \
                                                = Annotation('e_2', match.start(), matched_text)

    def __fix_issues(self, part):
        to_be_removed = []
        for index, ann in enumerate(part.predicted_annotations):
            start = ann.offset
            end = ann.offset + len(ann.text)

            # split multiple mentions
            if re.search(' *(/) *', ann.text):
                split = re.split(' *(/) *', ann.text)

                if self.at_least_one_letter_n_number_letter_n_number.search(split[0]) \
                        and self.at_least_one_letter_n_number_letter_n_number.search(split[2]):
                    to_be_removed.append(index)
                    part.predicted_annotations.append(Annotation(ann.class_id, ann.offset, split[0]))
                    part.predicted_annotations.append(
                        Annotation(ann.class_id, part.text.find(split[2], ann.offset), split[2]))

            # split multiple mentions
            if re.search(' *(and|,|or) *', ann.text):
                to_be_removed.append(index)
                split = re.split(' *(and|,|or) *', ann.text)
                if split[0]:
                    part.predicted_annotations.append(Annotation(ann.class_id, ann.offset, split[0]))
                if split[2]:
                    part.predicted_annotations.append(
                        Annotation(ann.class_id, part.text.find(split[2], ann.offset), split[2]))

            # fix boundary #17000021	251	258	1858C>T --> +1858C>T
            if re.search('^[0-9]', ann.text) and re.search('([\-\+])', part.text[start - 1]):
                ann.offset -= 1
                ann.text = part.text[start - 1] + ann.text

            # fix boundary delete (
            if ann.text[0] == '(' and ')' not in ann.text:
                ann.offset += 1
                ann.text = ann.text[1:]

            # fix boundary delete )
            if ann.text[-1] == ')' and '(' not in ann.text:
                ann.text = ann.text[:-1]

            # fix boundary add missing (
            if part.text[start - 1] == '(' and ')' in ann.text:
                ann.offset -= 1
                ann.text = '(' + ann.text

            # fix boundary add missing )
            try:
                if part.text[end] == ')' and '(' in ann.text:
                    ann.text += ')'
            except IndexError:
                pass
        part.predicted_annotations = [ann for index, ann in enumerate(part.predicted_annotations)
                                      if index not in to_be_removed]

def construct_regex_patterns_from_predictions(dataset):
    """
    :type dataset: nala.structures.data.Dataset
    """
    regex_patterns = []
    for ann in dataset.predicted_annotations():
        item = ann.text
        # escape special regex characters
        item = item.replace('.', '\.').replace('+', '\+').replace(')', '\)').replace('(', '\(')

        # numbers pattern
        item = re.sub('[0-9]+', '[0-9]+', item)

        # take care of special tokens
        item = re.sub('(IVS|EX)', '@@@@', item)
        item = re.sub('(rs|ss)', '@@@', item)

        # letters pattern
        item = re.sub('[a-z]', '[a-z]', item)
        item = re.sub('[A-Z]', '[A-Z]', item)

        # revert back special tokens
        item = re.sub('@@@@', '(IVS|EX)', item)
        item = re.sub('@@@', '(rs|ss)', item)

        regex_patterns.append(re.compile(item))

    # include already prepared regex patterns
    with open(pkg_resources.resource_filename('nala.data', 'RegEx.NL')) as file:
        for regex in csv.reader(file, delimiter='\t'):
            if regex[0].startswith('(?-xism:'):
                try:
                    regex_patterns.append(re.compile(regex[0].replace('(?-xism:', ''),
                                                     re.VERBOSE | re.IGNORECASE | re.DOTALL | re.MULTILINE))
                except:
                    pass
            else:
                regex_patterns.append(re.compile(regex[0]))

    # add our own custom regex
    regex_patterns.append(re.compile('[ATCG][0-9]+[ATCG]/[ATCG]'))

    return regex_patterns
<commit_msg>Fix bug in  post-processing<commit_after>import pkg_resources
import csv
import re
from nala.learning.evaluators import is_overlapping
from nala.structures.data import Annotation

class PostProcessing:
    def __init__(self):
        self.at_least_one_letter_n_number_letter_n_number = re.compile('(?=.*[A-Za-z])(?=.*[0-9])[A-Za-z0-9]+')
        self.short = re.compile('^[A-Z][0-9][A-Z]$')

    def process(self, dataset):
        existing_predictions = []

        for doc_id, doc in dataset.documents.items():
            for part_id, part in doc.parts.items():
                self.__fix_issues(part)

                for index, ann in enumerate(part.predicted_annotations):
                    existing_predictions.append((ann.offset, ann.offset + len(ann.text), part_id, ann.class_id, doc_id, index))

        regex_patterns = construct_regex_patterns_from_predictions(dataset)

        for doc_id, doc in dataset.documents.items():
            for part_id, part in doc.parts.items():
                for regex in regex_patterns:
                    for match in regex.finditer(part.text):
                        offset = (match.start(), match.end(), part_id, 'e_2', doc_id)
                        matched_text = part.text[match.start():match.end()]

                        # TODO Refactor into regex instead of check
                        try:
                            space_before = part.text[match.start() - 1] == ' '
                        except IndexError:
                            space_before = True
                        try:
                            space_after = part.text[match.end()] == ' '
                        except IndexError:
                            space_after = True

                        if not is_overlapping(offset, existing_predictions):
                            if not self.short.search(matched_text) and space_before and space_after \
                                    and self.at_least_one_letter_n_number_letter_n_number.search(matched_text):
                                existing_predictions.append(offset)
                                part.predicted_annotations.append(Annotation('e_2', match.start(), matched_text))
                        else:
                            # TODO Refactor to return an object
                            # our custom part (needs to be optimized) adds 1% to the f_measure (eg. 87 to 88)
                            for offset_b in existing_predictions:
                                # if there is a partial overlap with a regex match
                                if offset[2:5] == offset_b[2:5] and offset[0] <= offset_b[1] and offset[1] >= offset_b[0]:
                                    # but that overlap does not contain spaces and cover text span bigger in length
                                    if (offset[1] - offset[0]) > (offset_b[1]-offset_b[0]) and ' ' not in matched_text:
                                        # and additionally it's between two spaces
                                        if re.search(' +{} +'.format(regex.pattern), part.text):
                                            # replace the existing one by the one found one since it is probably better
                                            dataset.documents[offset_b[-2]].parts[offset_b[-4]].predicted_annotations[offset_b[-1]] \
                                                = Annotation('e_2', match.start(), matched_text)

    def __fix_issues(self, part):
        to_be_removed = []
        for index, ann in enumerate(part.predicted_annotations):
            start = ann.offset
            end = ann.offset + len(ann.text)

            # split multiple mentions
            if re.search(' *(/) *', ann.text):
                split = re.split(' *(/) *', ann.text)

                if self.at_least_one_letter_n_number_letter_n_number.search(split[0]) \
                        and self.at_least_one_letter_n_number_letter_n_number.search(split[2]):
                    to_be_removed.append(index)
                    part.predicted_annotations.append(Annotation(ann.class_id, ann.offset, split[0]))
                    part.predicted_annotations.append(
                        Annotation(ann.class_id, part.text.find(split[2], ann.offset), split[2]))

            # split multiple mentions
            if re.search(' *(and|,|or) *', ann.text):
                to_be_removed.append(index)
                split = re.split(' *(and|,|or) *', ann.text)
                if split[0]:
                    part.predicted_annotations.append(Annotation(ann.class_id, ann.offset, split[0]))
                if split[2]:
                    part.predicted_annotations.append(
                        Annotation(ann.class_id, part.text.find(split[2], ann.offset), split[2]))

            # fix boundary #17000021	251	258	1858C>T --> +1858C>T
            if re.search('^[0-9]', ann.text) and re.search('([\-\+])', part.text[start - 1]):
                ann.offset -= 1
                ann.text = part.text[start - 1] + ann.text

            # fix boundary delete (
            if ann.text[0] == '(' and ')' not in ann.text:
                ann.offset += 1
                ann.text = ann.text[1:]

            # fix boundary delete )
            if ann.text[-1] == ')' and '(' not in ann.text:
                ann.text = ann.text[:-1]

            # fix boundary add missing (
            if part.text[start - 1] == '(' and ')' in ann.text:
                ann.offset -= 1
                ann.text = '(' + ann.text

            # fix boundary add missing )
            try:
                if part.text[end] == ')' and '(' in ann.text:
                    ann.text += ')'
            except IndexError:
                pass
        part.predicted_annotations = [ann for index, ann in enumerate(part.predicted_annotations)
                                      if index not in to_be_removed]

def construct_regex_patterns_from_predictions(dataset):
    """
    :type dataset: nala.structures.data.Dataset
    """
    regex_patterns = []
    for ann in dataset.predicted_annotations():
        item = ann.text
        # escape special regex characters
        item = item.replace('.', '\.').replace('+', '\+').replace(')', '\)').replace('(', '\(').replace('*', '\*')

        # numbers pattern
        item = re.sub('[0-9]+', '[0-9]+', item)

        # take care of special tokens
        item = re.sub('(IVS|EX)', '@@@@', item)
        item = re.sub('(rs|ss)', '@@@', item)

        # letters pattern
        item = re.sub('[a-z]', '[a-z]', item)
        item = re.sub('[A-Z]', '[A-Z]', item)

        # revert back special tokens
        item = re.sub('@@@@', '(IVS|EX)', item)
        item = re.sub('@@@', '(rs|ss)', item)

        regex_patterns.append(re.compile(item))

    # include already prepared regex patterns
    with open(pkg_resources.resource_filename('nala.data', 'RegEx.NL')) as file:
        for regex in csv.reader(file, delimiter='\t'):
            if regex[0].startswith('(?-xism:'):
                try:
                    regex_patterns.append(re.compile(regex[0].replace('(?-xism:', ''),
                                                     re.VERBOSE | re.IGNORECASE | re.DOTALL | re.MULTILINE))
                except:
                    pass
            else:
                regex_patterns.append(re.compile(regex[0]))

    # add our own custom regex
    regex_patterns.append(re.compile('[ATCG][0-9]+[ATCG]/[ATCG]'))

    return regex_patterns
<|endoftext|>


<commit_before>from .environment import *
from .store import *
from .value import *

import viper.parser.ast.nodes as ns

from viper.parser.ast.nodes import AST

from typing import NamedTuple, Tuple, Union

STMTS = [
    ns.Stmt, ns.PlainStmt, ns.IfStmt, ns.ElifStmt, ns.ElseStmt, ns.Definition, ns.Parameter, ns.Arguments, ns.StmtBlock,
]

EXPRS = [
    ns.Expr, ns.IfExpr, ns.ElifExpr, ns.ElseExpr, ns.TestExprList, ns.TestExpr, ns.OrTestExpr, ns.AndTestExpr,
    ns.NotTestExpr, ns.OpExpr, ns.SubOpExpr, ns.AtomExpr, ns.Atom, ns.Trailer, ns.ExprBlock,
    ns.LhsExpr,
]

class EvalResult:
    pass

class EvalStmtResult(NamedTuple, EvalResult):
    env: Environment
    store: Store

class EvalExprResult(NamedTuple, EvalResult):
    val: Value
    store: Store

class EvalLhsResult(NamedTuple, EvalResult):
    maybe_env: Union[Environment, None]
    store: Store

def start_eval(code: AST, env: Environment = None, store: Store = None) -> EvalResult:
    if env is None:
        env = empty_env()
    if store is None:
        store = empty_store()
    if any(map(lambda s: isinstance(code, s), STMTS)):
        return eval_stmt(code, env, store)
    elif any(map(lambda e: isinstance(code, e), EXPRS)):
        return eval_expr(code, env, store)
    else:
        raise NotImplementedError(f"No evaluation rules for ASTs of type: {type(code).__name__}")

def eval_stmt(stmt: AST, env: Environment, store: Store) -> EvalStmtResult:
    if isinstance(stmt, ns.SimpleStmt):
        return eval_stmt(stmt.stmt, env, store)
    elif isinstance(stmt, ns.AssignStmt):
        # Evaluate right-hand side first.
        val, store = eval_expr(stmt.expr, env, store)
        # Pass the value to the left-hand side evaluation.
        maybe_env, store = eval_lhs_expr(stmt.lhs, env, store, val)
        if maybe_env is not None:
            env = maybe_env
        return EvalStmtResult(env, store)
    elif isinstance(stmt, ns.EmptyStmt):
        return EvalStmtResult(env, store)
    elif isinstance(stmt, ns.IfStmt):
        val, store2 = eval_expr(stmt.cond, env, store)
        if isinstance(val, TrueVal):
            return eval_stmt(stmt.then_body, env, store2)
        elif isinstance(val, FalseVal):
            for elif_stmt in stmt.elif_stmts:
                val, store2 = eval_expr(elif_stmt.cond, env, store)
                if isinstance(val, TrueVal):
                    return eval_stmt(elif_stmt.elif_body, env, store2)
            if stmt.else_stmt is not None:
                return eval_stmt(stmt.else_stmt.else_body, env, store)
            else:
                return EvalStmtResult(env, store)
        else:
            raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
    elif isinstance(stmt, ns.SimpleStmtBlock):
        return eval_stmt(stmt.stmt, env, store)
    elif isinstance(stmt, ns.CompoundStmtBlock):
        for sub_stmt in stmt.stmts:
            env, store = eval_stmt(sub_stmt, env, store)
        return EvalStmtResult(env, store)
    else:
        raise NotImplementedError(f"No implementation for statement type: {type(stmt).__name__}")

def eval_expr(expr: AST, env: Environment, store: Store) -> EvalExprResult:
    if isinstance(expr, ns.IfExpr):
        raise NotImplementedError
    elif isinstance(expr, ns.TestExprList):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            return EvalExprResult(TupleVal(*values), store)
    elif isinstance(expr, ns.TestExpr):
        return eval_expr(expr.test, env, store)
    elif isinstance(expr, ns.OrTestExpr):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            for val in values:
                if not isinstance(val, BoolVal):
                    raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
                if isinstance(val, TrueVal):
                    return EvalExprResult(TrueVal(), store)
            return EvalExprResult(FalseVal(), store)
    elif isinstance(expr, ns.AndTestExpr):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            for val in values:
                if not isinstance(val, BoolVal):
                    raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
                if isinstance(val, FalseVal):
                    return EvalExprResult(FalseVal(), store)
            return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.NegatedTestExpr):
        val, store = eval_expr(expr.op_expr, env, store)
        if not isinstance(val, BoolVal):
            raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
        if isinstance(val, TrueVal):
            return EvalExprResult(FalseVal(), store)
        else:
            return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.NotNegatedTestExpr):
        return eval_expr(expr.op_expr, env, store)
    elif isinstance(expr, ns.OpExpr):
        if expr.left_op is not None:
            raise NotImplementedError
        if expr.right_op is not None:
            raise NotImplementedError
        if expr.sub_op_exprs:
            raise NotImplementedError
        return eval_expr(expr.atom, env, store)
    elif isinstance(expr, ns.AtomExpr):
        if expr.trailers:
            raise NotImplementedError
        return eval_expr(expr.atom, env, store)
    elif isinstance(expr, ns.ParenAtom):
        if expr.tests is None:
            return EvalExprResult(UnitVal(), store)
        else:
            return eval_expr(expr.tests, env, store)
    elif isinstance(expr, ns.NameAtom):
        name = expr.name.text
        if name in env:
            return EvalExprResult(store[env[name]], store)
        else:
            raise RuntimeError(f"No such name in environment: {name}")
    elif isinstance(expr, ns.NumberAtom):
        return EvalExprResult(NumVal(expr.num.text), store)
    elif isinstance(expr, ns.EllipsisAtom):
        return EvalExprResult(EllipsisVal(), store)
    elif isinstance(expr, ns.TrueAtom):
        return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.FalseAtom):
        return EvalExprResult(FalseVal(), store)
    else:
        raise NotImplementedError

def eval_lhs_expr(expr: AST, env: Environment, store: Store, val: Value) -> EvalLhsResult:
    if isinstance(expr, ns.Pattern):
        return eval_pattern(expr, env, store, val)
    else:
        raise NotImplementedError

def eval_pattern(ptrn: ns.Pattern, env: Environment, store: Store, val: Value) -> EvalLhsResult:
    if isinstance(ptrn, ns.TypedVariablePattern):
        ...
    elif isinstance(ptrn, ns.TypedAnonymousPattern):
        ...
    elif isinstance(ptrn, ns.TypedFieldPattern):
        ...
    elif isinstance(ptrn, ns.SimpleVariablePattern):
        name = ptrn.id.id.text
        env, store = bind_val(name, val, env, store)
        return EvalLhsResult(env, store)
    elif isinstance(ptrn, ns.SimpleAnonymousPattern):
        return EvalLhsResult(env, store)
    elif isinstance(ptrn, ns.SimpleFieldPattern):
        ...
    elif isinstance(ptrn, ns.SimpleParenPattern):
        ...
    elif isinstance(ptrn, ns.PatternList):
        ...
    else:
        raise NotImplementedError

def bind_val(name: str, val: Value, env: Environment, store: Store) -> Tuple[Environment, Store]:
    if name in env:
        store = extend_store(store, val, env[name])
    else:
        env = extend_env(env, name, store.next_addr)
        store = extend_store(store, val)
    return env, store
<commit_msg>Add some method docstrings for clarity<commit_after>from .environment import *
from .store import *
from .value import *

import viper.parser.ast.nodes as ns

from viper.parser.ast.nodes import AST

from typing import NamedTuple, Tuple, Union

STMTS = [
    ns.Stmt, ns.PlainStmt, ns.IfStmt, ns.ElifStmt, ns.ElseStmt, ns.Definition, ns.Parameter, ns.Arguments, ns.StmtBlock,
]

EXPRS = [
    ns.Expr, ns.IfExpr, ns.ElifExpr, ns.ElseExpr, ns.TestExprList, ns.TestExpr, ns.OrTestExpr, ns.AndTestExpr,
    ns.NotTestExpr, ns.OpExpr, ns.SubOpExpr, ns.AtomExpr, ns.Atom, ns.Trailer, ns.ExprBlock,
    ns.LhsExpr,
]

class EvalResult:
    pass

class EvalStmtResult(NamedTuple, EvalResult):
    env: Environment
    store: Store

class EvalExprResult(NamedTuple, EvalResult):
    val: Value
    store: Store

class EvalLhsResult(NamedTuple, EvalResult):
    maybe_env: Union[Environment, None]
    store: Store

def start_eval(code: AST, env: Environment = None, store: Store = None) -> EvalResult:
    if env is None:
        env = empty_env()
    if store is None:
        store = empty_store()
    if any(map(lambda s: isinstance(code, s), STMTS)):
        return eval_stmt(code, env, store)
    elif any(map(lambda e: isinstance(code, e), EXPRS)):
        return eval_expr(code, env, store)
    else:
        raise NotImplementedError(f"No evaluation rules for ASTs of type: {type(code).__name__}")

def eval_stmt(stmt: AST, env: Environment, store: Store) -> EvalStmtResult:
    if isinstance(stmt, ns.SimpleStmt):
        return eval_stmt(stmt.stmt, env, store)
    elif isinstance(stmt, ns.AssignStmt):
        # Evaluate right-hand side first.
        val, store = eval_expr(stmt.expr, env, store)
        # Pass the value to the left-hand side evaluation.
        maybe_env, store = eval_lhs_expr(stmt.lhs, env, store, val)
        if maybe_env is not None:
            env = maybe_env
        return EvalStmtResult(env, store)
    elif isinstance(stmt, ns.EmptyStmt):
        return EvalStmtResult(env, store)
    elif isinstance(stmt, ns.IfStmt):
        val, store2 = eval_expr(stmt.cond, env, store)
        if isinstance(val, TrueVal):
            return eval_stmt(stmt.then_body, env, store2)
        elif isinstance(val, FalseVal):
            for elif_stmt in stmt.elif_stmts:
                val, store2 = eval_expr(elif_stmt.cond, env, store)
                if isinstance(val, TrueVal):
                    return eval_stmt(elif_stmt.elif_body, env, store2)
            if stmt.else_stmt is not None:
                return eval_stmt(stmt.else_stmt.else_body, env, store)
            else:
                return EvalStmtResult(env, store)
        else:
            raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
    elif isinstance(stmt, ns.SimpleStmtBlock):
        return eval_stmt(stmt.stmt, env, store)
    elif isinstance(stmt, ns.CompoundStmtBlock):
        for sub_stmt in stmt.stmts:
            env, store = eval_stmt(sub_stmt, env, store)
        return EvalStmtResult(env, store)
    else:
        raise NotImplementedError(f"No implementation for statement type: {type(stmt).__name__}")

def eval_expr(expr: AST, env: Environment, store: Store) -> EvalExprResult:
    if isinstance(expr, ns.IfExpr):
        raise NotImplementedError
    elif isinstance(expr, ns.TestExprList):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            return EvalExprResult(TupleVal(*values), store)
    elif isinstance(expr, ns.TestExpr):
        return eval_expr(expr.test, env, store)
    elif isinstance(expr, ns.OrTestExpr):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            for val in values:
                if not isinstance(val, BoolVal):
                    raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
                if isinstance(val, TrueVal):
                    return EvalExprResult(TrueVal(), store)
            return EvalExprResult(FalseVal(), store)
    elif isinstance(expr, ns.AndTestExpr):
        values = []
        for test in expr.tests:
            val, store = eval_expr(test, env, store)
            values.append(val)
        if len(values) == 1:
            return EvalExprResult(values[0], store)
        else:
            for val in values:
                if not isinstance(val, BoolVal):
                    raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
                if isinstance(val, FalseVal):
                    return EvalExprResult(FalseVal(), store)
            return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.NegatedTestExpr):
        val, store = eval_expr(expr.op_expr, env, store)
        if not isinstance(val, BoolVal):
            raise RuntimeError(f"Not a boolean value: {val}")  # TODO: Use a custom error.
        if isinstance(val, TrueVal):
            return EvalExprResult(FalseVal(), store)
        else:
            return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.NotNegatedTestExpr):
        return eval_expr(expr.op_expr, env, store)
    elif isinstance(expr, ns.OpExpr):
        if expr.left_op is not None:
            raise NotImplementedError
        if expr.right_op is not None:
            raise NotImplementedError
        if expr.sub_op_exprs:
            raise NotImplementedError
        return eval_expr(expr.atom, env, store)
    elif isinstance(expr, ns.AtomExpr):
        if expr.trailers:
            raise NotImplementedError
        return eval_expr(expr.atom, env, store)
    elif isinstance(expr, ns.ParenAtom):
        if expr.tests is None:
            return EvalExprResult(UnitVal(), store)
        else:
            return eval_expr(expr.tests, env, store)
    elif isinstance(expr, ns.NameAtom):
        name = expr.name.text
        if name in env:
            return EvalExprResult(store[env[name]], store)
        else:
            raise RuntimeError(f"No such name in environment: {name}")
    elif isinstance(expr, ns.NumberAtom):
        return EvalExprResult(NumVal(expr.num.text), store)
    elif isinstance(expr, ns.EllipsisAtom):
        return EvalExprResult(EllipsisVal(), store)
    elif isinstance(expr, ns.TrueAtom):
        return EvalExprResult(TrueVal(), store)
    elif isinstance(expr, ns.FalseAtom):
        return EvalExprResult(FalseVal(), store)
    else:
        raise NotImplementedError

def eval_lhs_expr(expr: AST, env: Environment, store: Store, val: Value) -> EvalLhsResult:
    """
    Attempts to perform a binding of a value. If the shape of the value and the shape of the left-hand side
    are compatible, the binding is performed. An environment and store are returned. If the shapes are not
    compatible, only the store is returned and the environment will be a None.

    :param expr: a left-hand side expression
    :param env: an environment
    :param store: a store
    :param val: a value to bind
    :return: a tuple of a Maybe(env) (as an Environment or a None) and a store
    """
    if isinstance(expr, ns.Pattern):
        return eval_pattern(expr, env, store, val)
    else:
        raise NotImplementedError

def eval_pattern(ptrn: ns.Pattern, env: Environment, store: Store, val: Value) -> EvalLhsResult:
    """
    Binds a value to a pattern, if the value's type and the pattern's shape align.

    :param ptrn: a pattern to match against
    :param env: an environment
    :param store: a store
    :param val: a value to bind
    :return: a tuple of a Maybe(env) (as an Environment or a None) and a store
    """
    if isinstance(ptrn, ns.TypedVariablePattern):
        ...
    elif isinstance(ptrn, ns.TypedAnonymousPattern):
        ...
    elif isinstance(ptrn, ns.TypedFieldPattern):
        ...
    elif isinstance(ptrn, ns.SimpleVariablePattern):
        name = ptrn.id.id.text
        env, store = bind_val(name, val, env, store)
        return EvalLhsResult(env, store)
    elif isinstance(ptrn, ns.SimpleAnonymousPattern):
        return EvalLhsResult(env, store)
    elif isinstance(ptrn, ns.SimpleFieldPattern):
        ...
    elif isinstance(ptrn, ns.SimpleParenPattern):
        ...
    elif isinstance(ptrn, ns.PatternList):
        ...
    else:
        raise NotImplementedError

def bind_val(name: str, val: Value, env: Environment, store: Store) -> Tuple[Environment, Store]:
    """
    Binds a value to a name. If the name exists in the environment, the previous binding is overwritten.
    Otherwise, a new binding is created.
    """
    if name in env:
        store = extend_store(store, val, env[name])
    else:
        env = extend_env(env, name, store.next_addr)
        store = extend_store(store, val)
    return env, store
<|endoftext|>


<commit_before>#!/usr/bin/python

from .naive import StratNaive
import random
import numpy as np
import copy

from ..ngmeth_utils.srtheo_utils import srtheo_voc
import copy

class StratTSRMax(StratNaive):

	def __init__(self, vu_cfg, efficient_computing=False, mem_type='interaction_counts_sliding_window_local',time_scale=10,cache=False,**strat_cfg2):
		StratNaive.__init__(self,vu_cfg=vu_cfg, **copy.deepcopy(strat_cfg2))
		mp = {'mem_type':mem_type,'time_scale':time_scale}
		if 'interaction_count' not in [ mmpp['mem_type'][:17] for mmpp in self.memory_policies]:
			self.memory_policies.append(mp)
		if cache:
			assert 'proba_of_success_increase' not in [ mmpp['mem_type'][:25] for mmpp in self.memory_policies]
			self.memory_policies.append({'mem_type':'proba_of_success_increase'})
		self.efficient_computing = efficient_computing

	def pick_m(self,voc,mem,context):

		m_list = self.get_mval_list(voc=voc,mem=mem,context=context)
		val_max = None

		for key,value in m_list:
			if val_max is None or value > val_max:
				val_max = value
		max_list = []
		for key,value in m_list:
			if value == val_max:
				max_list.append(key)

		m = voc.get_random_m(max_list)

		if 'proba_of_success_increase' in list(mem.keys()):
			m_rm_list = set([m])
			for w2 in voc.get_known_words(m=m):
				m_rm_list = m_rm_list | set(voc.get_known_meanings(w=w2))
			for m2 in m_rm_list:
				if m2 in list(mem['proba_of_success_increase'].keys()):
					del mem['proba_of_success_increase'][m2]
		return m

	def hearer_pick_m(self,voc,mem,context):
		return self.pick_m(voc, mem,context)

	def get_mval_list(self,voc,mem,context):
		m_list = []

		m_explo = None
		if len(voc.get_unknown_meanings())>0:
			m_explo = voc.get_new_unknown_m()
			w_explo = voc.get_new_unknown_w()
			mm_list = voc.get_known_meanings() + [m_explo]
		else:
			mm_list = voc.get_known_meanings()

		global_mat = mem['interact_count_voc']
			#m_explo = None
			#if len(voc.get_unknown_meanings())>0:
			#	m_explo = voc.get_new_unknown_m()
			#	w_explo = voc.get_new_unknown_w()
			#	mm_list = voc.get_known_meanings() + [m_explo]
			#else:
			#	mm_list = voc.get_known_meanings()
		print('m',mm_list,m_explo)
		for m1 in mm_list:
			if 'proba_of_success_increase' in list(mem.keys()) and m1 in list(mem['proba_of_success_increase'].keys()):
				val_norm = mem['proba_of_success_increase'][m1]
				m_list.append((m1,val_norm))
			else:
				val = 0
				if m1 == m_explo:
					ww_list = [w_explo]
				else:
					ww_list = voc.get_known_words(m=m1)
				print(ww_list)
				for w in ww_list:
					p_success = float(global_mat.get_value(m1,w,content_type='w'))
					p_fail = 1. - p_success
					if hasattr(voc,'get_alterable_shallow_copy'):
						voc_success = voc.get_alterable_shallow_copy()
					else:
						voc_success = copy.deepcopy(voc)
					self.update_speaker(m1, w, m1, voc_success, mem, bool_succ=True, context=context)
					mem_success = mem.simulated_update_memory(ms=m1,w=w,mh=m1,voc=voc_success,role='speaker',bool_succ=True,context=context)
											#global_mat_2 = mem_success['success_mw']+mem_success['fail_mw']

					global_mat_2 = mem_success['interact_count_voc']
											#dS_success = srtheo_voc(voc_success._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
					#if self.efficient_computing:
					#	dS_success = 0
					#	for m_loop in list(set(voc.get_known_meanings(w=w))|set([m1])):
					#		dS_success += 0.5 * (srtheo_voc(voc_success,m=m_loop,voc2=global_mat_2,role='speaker')/(voc_success.get_M()) - srtheo_voc(voc,m=m_loop,voc2=global_mat,role='speaker')/(voc.get_M()))
					#	for w_loop in list(set(voc.get_known_words(m=m1))|set([w])):
					#		dS_success += 0.5 * (srtheo_voc(voc_success,w=w_loop,voc2=global_mat_2,role='hearer') - srtheo_voc(voc,w=w_loop,voc2=global_mat,role='hearer'))
					#else:
					dS_success = srtheo_voc(voc_success,voc2=global_mat_2) - srtheo_voc(voc,voc2=global_mat)
					if hasattr(voc,'get_alterable_shallow_copy'):
						voc_fail = voc.get_alterable_shallow_copy()
					else:
						voc_fail = copy.deepcopy(voc)
					self.update_speaker(m1, w, m1, voc_fail, mem, bool_succ=False, context=context)
					mem_fail = mem.simulated_update_memory(ms=m1,w=w,mh=m1,voc=voc_fail,role='speaker',bool_succ=False,context=context)
								#global_mat_2 = mem_fail['success_mw']+mem_fail['fail_mw']
					global_mat_2 = mem_fail['interact_count_voc']
										#dS_fail = srtheo_voc(voc_fail._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
											#dS_success = srtheo_voc(voc_success._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
					#if self.efficient_computing:
					#	dS_fail = 0
					#	for m_loop in list(set(voc.get_known_meanings(w=w))|set([m1])):
					#		dS_fail += 0.5 * (srtheo_voc(voc_fail,m=m_loop,voc2=global_mat_2,role='speaker')/(voc_fail.get_M()) - srtheo_voc(voc,m=m_loop,voc2=global_mat,role='speaker')/(voc.get_M()))
					#	for w_loop in list(set(voc.get_known_words(m=m1))|set([w])):
					#		dS_fail += 0.5 * (srtheo_voc(voc_fail,w=w_loop,voc2=global_mat_2,role='hearer') - srtheo_voc(voc,w=w_loop,voc2=global_mat,role='hearer'))
					#else:
					dS_fail = srtheo_voc(voc_fail,voc2=global_mat_2) - srtheo_voc(voc,voc2=global_mat)
					val += p_success * dS_success + p_fail * dS_fail
#				r = random.random()
#				if r < 1./1:
#					if m1 == m_explo:
#						print 'mexplo'
#					print val,p_success,dS_success,dS_fail
#
#					print  srtheo_voc(voc_success,voc2=global_mat_2) , srtheo_voc(voc,voc2=global_mat)
				if m1 == m_explo:
					adj_poss_increase = 1.
					#factor = voc.get_M()/(voc.get_M()+adj_poss_increase)
					factor = 1.
					m_list.append((m_explo,val*factor))
				else:
					m_list.append((m1,val/len(voc.get_known_words(m=m1))))
					if 'proba_of_success_increase' in list(mem.keys()):
						mem['proba_of_success_increase'][m1] = float(val)/len(voc.get_known_words(m=m1))
#			if len(voc.get_unknown_meanings())>0:
#				m_explo = voc.get_new_unknown_m()
	#				w_explo = voc.get_new_unknown_w()
	#				mem_explo = mem.simulated_update_memory(ms=m_explo,w=w_explo,mh=m_explo,voc=voc,role='speaker',bool_succ=False,context=context)
	#
	#				m_list.append((m_explo,mem_val))

		return m_list

class StratTSRMaxMAB(StratTSRMax):

	def __init__(self, vu_cfg, efficient_computing=True, mem_type='interaction_counts_sliding_window_local',time_scale=10,cache=True,bandit_type='bandit',**strat_cfg2):
		StratTSRMax.__init__(self, vu_cfg=copy.deepcopy(vu_cfg), efficient_computing=efficient_computing, mem_type=mem_type,time_scale=time_scale,cache=cache, **copy.deepcopy(strat_cfg2))
		mp = {'mem_type':bandit_type}
		if 'bandit' not in [ mmpp['mem_type'][:6] for mmpp in self.memory_policies]:
			self.memory_policies.append(mp)

	def get_mval_list(self,voc,mem,context):
		m_list = []
		if len(voc.get_unknown_meanings())>0:
			m_explo = voc.get_new_unknown_m()
			val_explo = np.random.beta(mem['bandit']['arms']['arm_explo'][0],mem['bandit']['arms']['arm_explo'][1])
			m_list.append((m_explo,val_explo))

		for m in list(mem['bandit']['arms']['others'].keys()):
			val = np.random.beta(mem['bandit']['arms']['others'][m][0],mem['bandit']['arms']['others'][m][1])
			m_list.append((m,val))

		return m_list
<commit_msg>fixed lapsmax startegy<commit_after>#!/usr/bin/python

from .naive import StratNaive
import random
import numpy as np
import copy

from ..ngmeth_utils.srtheo_utils import srtheo_voc
import copy

class StratTSRMax(StratNaive):

	def __init__(self, vu_cfg, efficient_computing=False, mem_type='interaction_counts_sliding_window_local',time_scale=10,cache=False,**strat_cfg2):
		StratNaive.__init__(self,vu_cfg=vu_cfg, **copy.deepcopy(strat_cfg2))
		mp = {'mem_type':mem_type,'time_scale':time_scale}
		if 'interaction_count' not in [ mmpp['mem_type'][:17] for mmpp in self.memory_policies]:
			self.memory_policies.append(mp)
		if cache:
			assert 'proba_of_success_increase' not in [ mmpp['mem_type'][:25] for mmpp in self.memory_policies]
			self.memory_policies.append({'mem_type':'proba_of_success_increase'})
		self.efficient_computing = efficient_computing

	def pick_m(self,voc,mem,context):

		m_list = self.get_mval_list(voc=voc,mem=mem,context=context)
		val_max = None

		for key,value in m_list:
			if val_max is None or value > val_max:
				val_max = value
		max_list = []
		for key,value in m_list:
			if value == val_max:
				max_list.append(key)

		m = voc.get_random_m(max_list)

		if 'proba_of_success_increase' in list(mem.keys()):
			m_rm_list = set([m])
			for w2 in voc.get_known_words(m=m):
				m_rm_list = m_rm_list | set(voc.get_known_meanings(w=w2))
			for m2 in m_rm_list:
				if m2 in list(mem['proba_of_success_increase'].keys()):
					del mem['proba_of_success_increase'][m2]
		return m

	def hearer_pick_m(self,voc,mem,context):
		return self.pick_m(voc, mem,context)

	def get_mval_list(self,voc,mem,context):
		m_list = []

		m_explo = None
		if len(voc.get_unknown_meanings())>0:
			m_explo = voc.get_new_unknown_m()
			w_explo = voc.get_new_unknown_w()
			mm_list = voc.get_known_meanings() + [m_explo]
		else:
			mm_list = voc.get_known_meanings()

		global_mat = mem['interact_count_voc']
			#m_explo = None
			#if len(voc.get_unknown_meanings())>0:
			#	m_explo = voc.get_new_unknown_m()
			#	w_explo = voc.get_new_unknown_w()
			#	mm_list = voc.get_known_meanings() + [m_explo]
			#else:
			#	mm_list = voc.get_known_meanings()
		for m1 in mm_list:
			if 'proba_of_success_increase' in list(mem.keys()) and m1 in list(mem['proba_of_success_increase'].keys()):
				val_norm = mem['proba_of_success_increase'][m1]
				m_list.append((m1,val_norm))
			else:
				val = 0
				if m1 == m_explo:
					ww_list = [w_explo]
				else:
					ww_list = voc.get_known_words(m=m1)
				for w in ww_list:
					p_success = float(global_mat.get_value(m1,w,content_type='w'))
					p_fail = 1. - p_success
					if hasattr(voc,'get_alterable_shallow_copy'):
						voc_success = voc.get_alterable_shallow_copy()
					else:
						voc_success = copy.deepcopy(voc)
					self.update_speaker(m1, w, m1, voc_success, mem, bool_succ=True, context=context)
					mem_success = mem.simulated_update_memory(ms=m1,w=w,mh=m1,voc=voc_success,role='speaker',bool_succ=True,context=context)
											#global_mat_2 = mem_success['success_mw']+mem_success['fail_mw']

					global_mat_2 = mem_success['interact_count_voc']
											#dS_success = srtheo_voc(voc_success._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
					#if self.efficient_computing:
					#	dS_success = 0
					#	for m_loop in list(set(voc.get_known_meanings(w=w))|set([m1])):
					#		dS_success += 0.5 * (srtheo_voc(voc_success,m=m_loop,voc2=global_mat_2,role='speaker')/(voc_success.get_M()) - srtheo_voc(voc,m=m_loop,voc2=global_mat,role='speaker')/(voc.get_M()))
					#	for w_loop in list(set(voc.get_known_words(m=m1))|set([w])):
					#		dS_success += 0.5 * (srtheo_voc(voc_success,w=w_loop,voc2=global_mat_2,role='hearer') - srtheo_voc(voc,w=w_loop,voc2=global_mat,role='hearer'))
					#else:
					dS_success = srtheo_voc(voc_success,voc2=global_mat_2) - srtheo_voc(voc,voc2=global_mat)
					if hasattr(voc,'get_alterable_shallow_copy'):
						voc_fail = voc.get_alterable_shallow_copy()
					else:
						voc_fail = copy.deepcopy(voc)
					self.update_speaker(m1, w, m1, voc_fail, mem, bool_succ=False, context=context)
					mem_fail = mem.simulated_update_memory(ms=m1,w=w,mh=m1,voc=voc_fail,role='speaker',bool_succ=False,context=context)
								#global_mat_2 = mem_fail['success_mw']+mem_fail['fail_mw']
					global_mat_2 = mem_fail['interact_count_voc']
										#dS_fail = srtheo_voc(voc_fail._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
											#dS_success = srtheo_voc(voc_success._content,global_mat_2) - srtheo_voc(voc._content,global_mat)
					#if self.efficient_computing:
					#	dS_fail = 0
					#	for m_loop in list(set(voc.get_known_meanings(w=w))|set([m1])):
					#		dS_fail += 0.5 * (srtheo_voc(voc_fail,m=m_loop,voc2=global_mat_2,role='speaker')/(voc_fail.get_M()) - srtheo_voc(voc,m=m_loop,voc2=global_mat,role='speaker')/(voc.get_M()))
					#	for w_loop in list(set(voc.get_known_words(m=m1))|set([w])):
					#		dS_fail += 0.5 * (srtheo_voc(voc_fail,w=w_loop,voc2=global_mat_2,role='hearer') - srtheo_voc(voc,w=w_loop,voc2=global_mat,role='hearer'))
					#else:
					dS_fail = srtheo_voc(voc_fail,voc2=global_mat_2) - srtheo_voc(voc,voc2=global_mat)
					val += p_success * dS_success + p_fail * dS_fail
#				r = random.random()
#				if r < 1./1:
#					if m1 == m_explo:
#						print 'mexplo'
#					print val,p_success,dS_success,dS_fail
#
#					print  srtheo_voc(voc_success,voc2=global_mat_2) , srtheo_voc(voc,voc2=global_mat)
				if m1 == m_explo:
					adj_poss_increase = 1.
					#factor = voc.get_M()/(voc.get_M()+adj_poss_increase)
					factor = 1.
					m_list.append((m_explo,val*factor))
				else:
					m_list.append((m1,val/len(voc.get_known_words(m=m1))))
					if 'proba_of_success_increase' in list(mem.keys()):
						mem['proba_of_success_increase'][m1] = float(val)/len(voc.get_known_words(m=m1))
#			if len(voc.get_unknown_meanings())>0:
#				m_explo = voc.get_new_unknown_m()
	#				w_explo = voc.get_new_unknown_w()
	#				mem_explo = mem.simulated_update_memory(ms=m_explo,w=w_explo,mh=m_explo,voc=voc,role='speaker',bool_succ=False,context=context)
	#
	#				m_list.append((m_explo,mem_val))

		return m_list

class StratTSRMaxMAB(StratTSRMax):

	def __init__(self, vu_cfg, efficient_computing=True, mem_type='interaction_counts_sliding_window_local',time_scale=10,cache=True,bandit_type='bandit',**strat_cfg2):
		StratTSRMax.__init__(self, vu_cfg=copy.deepcopy(vu_cfg), efficient_computing=efficient_computing, mem_type=mem_type,time_scale=time_scale,cache=cache, **copy.deepcopy(strat_cfg2))
		mp = {'mem_type':bandit_type}
		if 'bandit' not in [ mmpp['mem_type'][:6] for mmpp in self.memory_policies]:
			self.memory_policies.append(mp)

	def get_mval_list(self,voc,mem,context):
		m_list = []
		if len(voc.get_unknown_meanings())>0:
			m_explo = voc.get_new_unknown_m()
			val_explo = np.random.beta(mem['bandit']['arms']['arm_explo'][0],mem['bandit']['arms']['arm_explo'][1])
			m_list.append((m_explo,val_explo))

		for m in list(mem['bandit']['arms']['others'].keys()):
			val = np.random.beta(mem['bandit']['arms']['others'][m][0],mem['bandit']['arms']['others'][m][1])
			m_list.append((m,val))

		return m_list
<|endoftext|>


<commit_before>#!/usr/bin/python

import json

DEFAULT_LEVELS="1, 6, 60"

def new_editors(old_period, new_period, levels=DEFAULT_LEVELS):
    # new editors list
    new_editors = []
    # Loading old period json filehandle
    json_old = json.load(old_period)
    data_old = json_old["result"]["Individual Results"][0]
    # iteration on each user of individual results list
    #       counts editor without edit on old period and add them to new_editors
    for k in data_old.keys():
        if(int(data_old[k]["edits"])==0):
            new_editors.append(k)

    json_new= json.load(new_period)
    data_new= json_new["result"]["Individual Results"][0]
    level = levels.split(",")
    # count of surviving editors by level
    new_counts = dict()
    for k in new_editors:
        edits = int(data_new[k]["edits"])
        for l in level:
            if edits>=int(l):
                if int(l) in new_counts.keys():
                    new_counts[int(l)] = new_counts[int(l)]+1
                else:
                    new_counts[int(l)] = 1
    print "%s - %s" % (json_new["parameters"]["start_date"], json_new["parameters"]["end_date"])
    for k in sorted(new_counts.keys()):
        print "\tnew editors with more than %d edits:\t %d" % (k, new_counts[k])

def getopt_fallback():
    """Fallback on getopt if your system don't have ArgumentParser."""
    import getopt, sys
    try:
        opts, args = getopt.getopt(sys.argv[1:], "o:n:l:", ["--old=", "--new=", "--levels="])
    except getopt.GetoptError:
        sys.exit(2)
    new = ""
    old = ""
    levels= ""
    for opt, arg in opts:
        if opt in ("-o", "--old"):
            old = arg
        if opt in ("-n", "--new"):
            new = arg
        if opt in ("-l", "--levels"):
            levels=arg
    if(new != "" and old != ""):
        old_period = open(old)
        new_period = open(new)
        if(levels==""):
            new_editors(old_period, new_period)
        else:
            new_editors(old_period, new_period, levels)
        old_period.close()
        new_period.close()

def main():
    """Main method, entry point of the script."""
    try:
        from argparse import ArgumentParser
        description = "Computes new editor numbers based on WikiMetrics data"
        parser = ArgumentParser(description=description)

        parser.add_argument("-o", "--old",
                            type=file,
                            dest="old_period",
                            metavar="old_period.json",
                            required=True,
                            help="The old period data, as a JSON file")

        parser.add_argument("-n", "--new",
                            type=file,
                            dest="new_period",
                            metavar="new_period.json",
                            required=True,
                            help="The new period data, as a JSON file")
        args = parser.parse_args()

        new_editors(args.old_period, args.new_period)
    except ImportError:
        getopt_fallback()

if __name__ == "__main__":
    main()
<commit_msg>Use Python list for levels handling in new_editors()<commit_after>#!/usr/bin/python

import json

DEFAULT_LEVELS=[1, 6, 60]

def new_editors(old_period, new_period, levels=DEFAULT_LEVELS):
    # new editors list
    new_editors = []
    # Loading old period json filehandle
    json_old = json.load(old_period)
    data_old = json_old["result"]["Individual Results"][0]
    # iteration on each user of individual results list
    #       counts editor without edit on old period and add them to new_editors
    for k in data_old.keys():
        if(int(data_old[k]["edits"])==0):
            new_editors.append(k)

    json_new= json.load(new_period)
    data_new= json_new["result"]["Individual Results"][0]

    # count of surviving editors by level
    new_counts = dict()
    for k in new_editors:
        edits = int(data_new[k]["edits"])
        for level in levels:
            if edits>=level:
                if level in new_counts.keys():
                    new_counts[level] = new_counts[level]+1
                else:
                    new_counts[level] = 1
    print "%s - %s" % (json_new["parameters"]["start_date"], json_new["parameters"]["end_date"])
    for k in sorted(new_counts.keys()):
        print "\tnew editors with more than %d edits:\t %d" % (k, new_counts[k])

def getopt_fallback():
    """Fallback on getopt if your system don't have ArgumentParser."""
    import getopt, sys
    try:
        opts, args = getopt.getopt(sys.argv[1:], "o:n:l:", ["--old=", "--new=", "--levels="])
    except getopt.GetoptError:
        sys.exit(2)
    new = ""
    old = ""
    levels= ""
    for opt, arg in opts:
        if opt in ("-o", "--old"):
            old = arg
        if opt in ("-n", "--new"):
            new = arg
        if opt in ("-l", "--levels"):
            levels=arg
    if(new != "" and old != ""):
        old_period = open(old)
        new_period = open(new)
        if(levels==""):
            new_editors(old_period, new_period)
        else:
            new_editors(old_period, new_period, levels.split(","))
        old_period.close()
        new_period.close()

def main():
    """Main method, entry point of the script."""
    try:
        from argparse import ArgumentParser
        description = "Computes new editor numbers based on WikiMetrics data"
        parser = ArgumentParser(description=description)

        parser.add_argument("-o", "--old",
                            type=file,
                            dest="old_period",
                            metavar="old_period.json",
                            required=True,
                            help="The old period data, as a JSON file")

        parser.add_argument("-n", "--new",
                            type=file,
                            dest="new_period",
                            metavar="new_period.json",
                            required=True,
                            help="The new period data, as a JSON file")
        args = parser.parse_args()

        new_editors(args.old_period, args.new_period)
    except ImportError:
        getopt_fallback()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python3
from __future__ import print_function

import argparse
import datetime
import os
import sys
import json
import uuid
import logging

def get_year():
    now = datetime.datetime.now()
    return now.year

def writable_directory(path):
    if not os.path.isdir(path):
        raise argparse.ArgumentTypeError(
            'not an existing directory: {}'.format(path))
    if not os.access(path, os.W_OK):
        raise argparse.ArgumentTypeError(
            'not a writable directory: {}'.format(path))
    return path

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Create new cmd-oyster.')
    parser.add_argument('write_dir', type=writable_directory, help='Directory to write to.')
    parser.add_argument('invocation', help='Command invocation.')
    parser.add_argument(
        '-v',
        '--verbose',
        help='More verbose logging',
        dest="loglevel",
        default=logging.WARNING,
        action="store_const",
        const=logging.INFO,
    )
    parser.add_argument(
        '-d',
        '--debug',
        help='Enable debugging logs',
        action="store_const",
        dest="loglevel",
        const=logging.DEBUG,
    )
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)

    new_uuid = str(uuid.uuid4())

    oyster = \
    {
        "component-commands": [
            "<FIXME>"
        ],
        "copying": {
            "authors": [
                "<FIXME>"
            ],
            "license-name": "MIT (Expat) License",
            "license-url": "http://opensource.org/licenses/MIT",
            "year": get_year()
        },
        "description": {
            "verbose-description": "<FIXME>"
        },
        "invocations": [
            {
                "invocation-string": args.invocation
            }
        ],
        "uuid": new_uuid
    }

    new_filename = new_uuid + '.json'
    new_filepath = os.path.join(args.write_dir, new_filename)
    with open(new_filepath, 'w') as new_file:
        json.dump(oyster, new_file, indent=4, separators=(',', ': '), sort_keys=True)
        print('Created new CmdOyster:\n{}'.format(new_filepath))
<commit_msg>Make output terse and parseable.<commit_after>#! /usr/bin/env python

import argparse
import datetime
import os
import sys
import json
import uuid
import logging

def get_year():
    now = datetime.datetime.now()
    return now.year

def writable_directory(path):
    if not os.path.isdir(path):
        raise argparse.ArgumentTypeError(
            'not an existing directory: {}'.format(path))
    if not os.access(path, os.W_OK):
        raise argparse.ArgumentTypeError(
            'not a writable directory: {}'.format(path))
    return path

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Create new cmd-oyster.')
    parser.add_argument('write_dir', type=writable_directory, help='Directory to write to.')
    parser.add_argument('invocation', help='Command invocation.')
    parser.add_argument(
        '-v',
        '--verbose',
        help='More verbose logging',
        dest="loglevel",
        default=logging.WARNING,
        action="store_const",
        const=logging.INFO,
    )
    parser.add_argument(
        '-d',
        '--debug',
        help='Enable debugging logs',
        action="store_const",
        dest="loglevel",
        const=logging.DEBUG,
    )
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)

    new_uuid = str(uuid.uuid4())

    oyster = \
    {
        "component-commands": [
            "<FIXME>"
        ],
        "copying": {
            "authors": [
                "<FIXME>"
            ],
            "license-name": "MIT (Expat) License",
            "license-url": "http://opensource.org/licenses/MIT",
            "year": get_year()
        },
        "description": {
            "verbose-description": "<FIXME>"
        },
        "invocations": [
            {
                "invocation-string": args.invocation
            }
        ],
        "uuid": new_uuid
    }

    new_filename = new_uuid + '.json'
    new_filepath = os.path.join(args.write_dir, new_filename)
    with open(new_filepath, 'w') as new_file:
        json.dump(oyster, new_file, indent=4, separators=(',', ': '), sort_keys=True)
        sys.stdout.write('{}\n'.format(new_filepath))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2014, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

from ..base import (BaseApplicationBackend, BaseCanvasBackend,
                    BaseTimerBackend)
from .. import Application, Canvas
from ...util import logger

# -------------------------------------------------------------------- init ---

capability = dict(  # things that can be set by the backend
    title=True,  # But it only applies to the dummy window :P
    size=True,  # We cannot possibly say we dont, because Canvas always sets it
    position=True,  # Dito
    show=True,  # Note: we don't alow this, but all scripts call show ...
    vsync=False,
    resizable=True,  # Yes, you can set to not be resizable (it always is)
    decorate=False,
    fullscreen=False,
    context=True,
    multi_window=True,
    scroll=True,
    parent=False,
)

def _set_config(c):
    _app.backend_module._set_config(c)

# Create our "backend" backend; The toolkit that is going to provide a
# canvas (e.g. OpenGL context) so we can render images.
# Note that if IPython has already loaded a GUI backend, vispy is
# probably going to use that as well, because it prefers loaded backends.
try:
    _app = Application('default')  # explicitly use default (avoid using test-app)
except RuntimeError:
    _msg = 'ipynb-png backend relies on a core backend'
    available, testable, why_not = False, False, _msg
else:
    available, testable, why_not = True, True, None

# Use that backend's shared context
KEYMAP = _app.backend_module.KEYMAP
SharedContext = _app.backend_module.SharedContext

# ------------------------------------------------------------- application ---

# todo: maybe trigger something in JS on any of these methods?
class ApplicationBackend(BaseApplicationBackend):

    def __init__(self):
        BaseApplicationBackend.__init__(self)
        self._backend2 = _app._backend

    def _vispy_get_backend_name(self):
        realname = self._backend2._vispy_get_backend_name()
        return 'ipynb-png (via %s)' % realname

    def _vispy_process_events(self):
        return self._backend2._vispy_process_events()

    def _vispy_run(self):
        return self._backend2._vispy_run()

    def _vispy_quit(self):
        return self._backend2._vispy_quit()

    def _vispy_get_native_app(self):
        return self._backend2._vispy_get_native_app()

# ------------------------------------------------------------------ canvas ---

class CanvasBackend(BaseCanvasBackend):

    def __init__(self, *args, **kwargs):
        BaseCanvasBackend.__init__(self, capability, SharedContext)

        # Test kwargs
#         if kwargs['size']:
#             raise RuntimeError('ipynb-png Canvas is not resizable')
#         if kwargs['position']:
#             raise RuntimeError('ipynb-png Canvas is not positionable')
        if not kwargs['decorate']:
            raise RuntimeError('ipynb-png Canvas is not decoratable (or not)')
        if kwargs['vsync']:
            raise RuntimeError('ipynb-png Canvas does not support vsync')
        if kwargs['fullscreen']:
            raise RuntimeError('ipynb-png Canvas does not support fullscreen')

        # Create real canvas. It is a backend to this backend
        kwargs['autoswap'] = False
        canvas = Canvas(app=_app, **kwargs)
        self._backend2 = canvas.native

        # Connect to events of canvas to keep up to date with size and draws
        canvas.events.draw.connect(self._on_draw)
        canvas.events.resize.connect(self._on_resize)
        self._initialized = False

        # Show the widget
        canvas.show()
        # todo: hide that canvas

    @property
    def _vispy_context(self):
        """Context to return for sharing"""
        return self._backend2._vispy_context

    def _vispy_warmup(self):
        return self._backend2._vispy_warmup()

    def _vispy_set_current(self):
        return self._backend2._vispy_set_current()

    def _vispy_swap_buffers(self):
        return self._backend2._vispy_swap_buffers()

    def _vispy_set_title(self, title):
        return self._backend2._vispy_set_title(title)
        #logger.warn('IPython notebook canvas has not title.')

    def _vispy_set_size(self, w, h):
        #return self._backend2._vispy_set_size(w, h)
        logger.warn('IPython notebook canvas cannot be resized.')

    def _vispy_set_position(self, x, y):
        logger.warn('IPython notebook canvas cannot be repositioned.')

    def _vispy_set_visible(self, visible):
        #self._backend2._vispy_set_visible(visible)
        if not visible:
            logger.warn('IPython notebook canvas cannot be hidden.')

    def _vispy_update(self):
        return self._backend2._vispy_update()

    def _vispy_close(self):
        return self._backend2._vispy_close()
        # todo: can we close on IPython side?

    def _vispy_get_position(self):
        return 0, 0

    def _vispy_get_size(self):
        return self._backend2._vispy_get_size()

    def _on_resize(self, event=None):
        # Event handler that is called by the underlying canvas
        if self._vispy_canvas is None:
            return
        size = self._backend2._vispy_get_size()
        self._vispy_canvas.events.resize(size=size)

    def _on_draw(self, event=None):
        # Event handler that is called by the underlying canvas
        if self._vispy_canvas is None:
            return
        # Handle initialization
        if not self._initialized:
            self._initialized = True
            self._vispy_canvas.events.initialize()
            self._on_resize()
        # Normal behavior
        self._vispy_set_current()
        self._vispy_canvas.events.draw(region=None)
        # todo: send screenshot to js

    def _get_events_from_js(self):
        pass
        # todo: implement this
        # Below are the events that you want to inject into vispy
#         self._vispy_canvas.events.resize(size=(w, h))
#         self._vispy_canvas.events.close()
#         self._keyEvent(self._vispy_canvas.events.key_press, ev)
#         self._keyEvent(self._vispy_canvas.events.key_release, ev)
#
#         self._vispy_mouse_press(native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             button=BUTTONMAP.get(ev.button(), 0),
#             modifiers = self._modifiers(ev),
#             )
#
#         self._vispy_mouse_release(
#             native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             button=BUTTONMAP[ev.button()],
#             modifiers = self._modifiers(ev),
#             )
#
#         self._vispy_mouse_move(
#             native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             modifiers=self._modifiers(ev),
#             )
#
#         self._vispy_canvas.events.mouse_wheel(
#             native=ev,
#             delta=(deltax, deltay),
#             pos=(ev.pos().x(), ev.pos().y()),
#             modifiers=self._modifiers(ev),
#             )

# ------------------------------------------------------------------- timer ---

class TimerBackend(BaseTimerBackend):

    def __init__(self, vispy_timer):
        self._backend2 = _app.backend_module.TimerBackend(vispy_timer)

    def _vispy_start(self, interval):
        return self._backend2._vispy_start(interval)

    def _vispy_stop(self):
        return self._backend2._vispy_stop()

    def _vispy_timeout(self):
        return self._backend2._vispy_timeout()
<commit_msg>ipynb-app: fix style and not available<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2014, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

from ..base import (BaseApplicationBackend, BaseCanvasBackend,
                    BaseTimerBackend)
from .. import Application, Canvas
from ...util import logger

# -------------------------------------------------------------------- init ---

capability = dict(  # things that can be set by the backend
    title=True,  # But it only applies to the dummy window :P
    size=True,  # We cannot possibly say we dont, because Canvas always sets it
    position=True,  # Dito
    show=True,  # Note: we don't alow this, but all scripts call show ...
    vsync=False,
    resizable=True,  # Yes, you can set to not be resizable (it always is)
    decorate=False,
    fullscreen=False,
    context=True,
    multi_window=True,
    scroll=True,
    parent=False,
)

def _set_config(c):
    _app.backend_module._set_config(c)

# Create our "backend" backend; The toolkit that is going to provide a
# canvas (e.g. OpenGL context) so we can render images.
# Note that if IPython has already loaded a GUI backend, vispy is
# probably going to use that as well, because it prefers loaded backends.
try:
     # Explicitly use default (avoid using test-app)
    _app = Application('default')
except RuntimeError:
    _msg = 'ipynb-png backend relies on a core backend'
    available, testable, why_not = False, False, _msg
else:
    available, testable, why_not = True, True, None

    # Use that backend's shared context
    KEYMAP = _app.backend_module.KEYMAP
    SharedContext = _app.backend_module.SharedContext

# ------------------------------------------------------------- application ---

# todo: maybe trigger something in JS on any of these methods?
class ApplicationBackend(BaseApplicationBackend):

    def __init__(self):
        BaseApplicationBackend.__init__(self)
        self._backend2 = _app._backend

    def _vispy_get_backend_name(self):
        realname = self._backend2._vispy_get_backend_name()
        return 'ipynb-png (via %s)' % realname

    def _vispy_process_events(self):
        return self._backend2._vispy_process_events()

    def _vispy_run(self):
        return self._backend2._vispy_run()

    def _vispy_quit(self):
        return self._backend2._vispy_quit()

    def _vispy_get_native_app(self):
        return self._backend2._vispy_get_native_app()

# ------------------------------------------------------------------ canvas ---

class CanvasBackend(BaseCanvasBackend):

    def __init__(self, *args, **kwargs):
        BaseCanvasBackend.__init__(self, capability, SharedContext)

        # Test kwargs
#         if kwargs['size']:
#             raise RuntimeError('ipynb-png Canvas is not resizable')
#         if kwargs['position']:
#             raise RuntimeError('ipynb-png Canvas is not positionable')
        if not kwargs['decorate']:
            raise RuntimeError('ipynb-png Canvas is not decoratable (or not)')
        if kwargs['vsync']:
            raise RuntimeError('ipynb-png Canvas does not support vsync')
        if kwargs['fullscreen']:
            raise RuntimeError('ipynb-png Canvas does not support fullscreen')

        # Create real canvas. It is a backend to this backend
        kwargs['autoswap'] = False
        canvas = Canvas(app=_app, **kwargs)
        self._backend2 = canvas.native

        # Connect to events of canvas to keep up to date with size and draws
        canvas.events.draw.connect(self._on_draw)
        canvas.events.resize.connect(self._on_resize)
        self._initialized = False

        # Show the widget
        canvas.show()
        # todo: hide that canvas

    @property
    def _vispy_context(self):
        """Context to return for sharing"""
        return self._backend2._vispy_context

    def _vispy_warmup(self):
        return self._backend2._vispy_warmup()

    def _vispy_set_current(self):
        return self._backend2._vispy_set_current()

    def _vispy_swap_buffers(self):
        return self._backend2._vispy_swap_buffers()

    def _vispy_set_title(self, title):
        return self._backend2._vispy_set_title(title)
        #logger.warn('IPython notebook canvas has not title.')

    def _vispy_set_size(self, w, h):
        #return self._backend2._vispy_set_size(w, h)
        logger.warn('IPython notebook canvas cannot be resized.')

    def _vispy_set_position(self, x, y):
        logger.warn('IPython notebook canvas cannot be repositioned.')

    def _vispy_set_visible(self, visible):
        #self._backend2._vispy_set_visible(visible)
        if not visible:
            logger.warn('IPython notebook canvas cannot be hidden.')

    def _vispy_update(self):
        return self._backend2._vispy_update()

    def _vispy_close(self):
        return self._backend2._vispy_close()
        # todo: can we close on IPython side?

    def _vispy_get_position(self):
        return 0, 0

    def _vispy_get_size(self):
        return self._backend2._vispy_get_size()

    def _on_resize(self, event=None):
        # Event handler that is called by the underlying canvas
        if self._vispy_canvas is None:
            return
        size = self._backend2._vispy_get_size()
        self._vispy_canvas.events.resize(size=size)

    def _on_draw(self, event=None):
        # Event handler that is called by the underlying canvas
        if self._vispy_canvas is None:
            return
        # Handle initialization
        if not self._initialized:
            self._initialized = True
            self._vispy_canvas.events.initialize()
            self._on_resize()
        # Normal behavior
        self._vispy_set_current()
        self._vispy_canvas.events.draw(region=None)
        # todo: send screenshot to js

    def _get_events_from_js(self):
        pass
        # todo: implement this
        # Below are the events that you want to inject into vispy
#         self._vispy_canvas.events.resize(size=(w, h))
#         self._vispy_canvas.events.close()
#         self._keyEvent(self._vispy_canvas.events.key_press, ev)
#         self._keyEvent(self._vispy_canvas.events.key_release, ev)
#
#         self._vispy_mouse_press(native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             button=BUTTONMAP.get(ev.button(), 0),
#             modifiers = self._modifiers(ev),
#             )
#
#         self._vispy_mouse_release(
#             native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             button=BUTTONMAP[ev.button()],
#             modifiers = self._modifiers(ev),
#             )
#
#         self._vispy_mouse_move(
#             native=ev,
#             pos=(ev.pos().x(), ev.pos().y()),
#             modifiers=self._modifiers(ev),
#             )
#
#         self._vispy_canvas.events.mouse_wheel(
#             native=ev,
#             delta=(deltax, deltay),
#             pos=(ev.pos().x(), ev.pos().y()),
#             modifiers=self._modifiers(ev),
#             )

# ------------------------------------------------------------------- timer ---

class TimerBackend(BaseTimerBackend):

    def __init__(self, vispy_timer):
        self._backend2 = _app.backend_module.TimerBackend(vispy_timer)

    def _vispy_start(self, interval):
        return self._backend2._vispy_start(interval)

    def _vispy_stop(self):
        return self._backend2._vispy_stop()

    def _vispy_timeout(self):
        return self._backend2._vispy_timeout()
<|endoftext|>