<commit_before># Lior Hirschfeld
# JargonBot

# -- Imports --
import re
import pickle
import praw
import random

from datetime import datetime
from time import sleep
from define import getDefinition
from collections import Counter
from nltk.stem import *
from textblob import TextBlob
from sklearn import linear_model

# -- Setup Variables --
r = praw.Reddit('bot3')
stemmer = PorterStemmer()
responses = []

with open('ids.pickle', 'rb') as handle:
    try:
        ids = pickle.load(handle)
    except EOFError:
        ids = []
with open('languages.pickle', 'rb') as handle:
    languages = pickle.load(handle)
with open('count.txt', 'r') as handle:
    count = [line.split()[0] for line in handle.readlines()]
    countStemmed = [stemmer.stem(word) for word in count]

# Model Takes: [Word Popularity, Word Length, Comment Length]
# Models is a dictionary with a touple at each key containing:
# (linear regression, randomness rate)
with open('models.pickle', 'rb') as handle:
    try:
        models = pickle.load(handle)
    except EOFError:
        models = {}

# -- Methods --
def jargon(lim, rate, subs, ml=False):
    searchReddit(lim, rate, subs, ml)

# Search Reddit for words that need to be defined, and define them.
def searchReddit(lim, rate, subs, ml):
    while True:
        for sub in subs:
            searchSub(sub, lim, ml)
        with open('ids.pickle', 'wb') as handle:
            pickle.dump(ids, handle, protocol=pickle.HIGHEST_PROTOCOL)

        if ml:
            updateModels()
        sleep(rate)

# Update the machine learning model and save it.
def updateModels():
    global responses, models
    currentTime = datetime.now()
    oldResponses = [(currentTime - r["time"]).total_seconds() > 3600
                             for r in responses]
    responses = [(currentTime - r["time"]).total_seconds() < 3600
                             for r in responses]
    for r in oldResponses:
        result = 0
        url = "https://reddit.com/" + r["sID"] + "?comment=" + r["cID"]
        submission = reddit.get_submission(url=url)
        comment_queue = submission.comments[:]
        if comment_queue:
            com = comment_queue.pop(0)
            result += com.score
            comment_queue.extend(com.replies)
        while comment_queue:
            com = comment_queue.pop(0)
            text = TextBlob(com.text)
            result += text.sentiment.polarity * com.score
        models[r["sub"]][0].fit([[r["popularity"], r["wLength"], r["cLength"]]],
                              [result])

        # Update odds of random choice
        models[r]["sub"][1] *= 0.96
    with open('models.pickle', 'wb') as handle:
        pickle.dump(models, handle, protocol=pickle.HIGHEST_PROTOCOL)

# Search a sub for words that need to be defined, and define them.
def searchSub(sub, lim, ml):
    global models
    if sub not in languages:
        analyze(sub)

    subreddit = r.subreddit(sub)
    subWords = [pair[0] for pair in languages[sub].most_common(10000)]
    for submission in subreddit.hot(limit=lim):
        comment_queue = submission.comments[:]
        while comment_queue:
            com = comment_queue.pop(0)
            if not hasattr(com, 'body') or com.id in ids:
                continue
            for word in com.body.split():
                # Stem the word and check if it is rare enough to be defined.
                # Find the most similar word in count to the stemmed word.
                word = stemmer.stem(word)
                if word not in subWords:
                    for item in countStemmed:
                        if item == word:
                            word = item
                            break
                    if ml:
                        if sub not in models:
                            models[sub] = (linear_model.LinearRegression(), 1)
                            models[sub][0].fit([[1000000, 10, 10]], [10])
                        # If ML, after basic checks, predict using the model
                        # to decide whether to reply.
                        if word in count:
                            popularity = count.index(word)
                        else:
                            popularity = 1000000

                        info = {"popularity": popularity, "wLength": len(word),
                                "cLength": len(com.body), "cID": com.id,
                                "sID": submission.id, "sub": sub}

                        if popularity > 10000:
                            # Sometimes, randomly reply to train the model.
                            if random.random() < models[sub][1]:
                                reply(com, word, ml, info=info)
                            elif models[sub][0].predict([[info["popularity"],
                                    info["wLength"], info["cLength"]]]) > 0:
                                reply(com, word, ml, info=info)
                        break
                    else:
                        if word not in count[:80000]:
                            reply(com, word, ml)
                            break
            ids.append(com.id)
            comment_queue.extend(com.replies)

# Reply to a comment with a word definition.
def reply(com, word, ml, info=None):
    global responses
    print("Found Comment:" + com.id)
    reply = ""
    # Get the definition of the word (if it exists)
    result = getDefinition(word)

    if result != None:
        # A definition has been found.
        reply += """Definition of {}: {}.\n\n
                    *{}.*""".format(word.upper(), result[0].capitalize(),
                                    result[1].capitalize())
        if ml:
            reply += """
                    \n\n---------\n\n
                    I am a bot which attempts to define difficult words
                    automatically. I use machine learning to do this, and I can
                    use your feedback to improve. Feel free to leave a comment
                    to let me know what you thought of this definition!
                    """
        reply += """\n\nAll data from http://www.oed.com/. Please contact
                    /u/liortulip with any questions or concerns."""

        try:
            cID = com.reply(reply)

            if ml:
                info["time"] = datetime.now()
                info["cID"] = cID
                responses.append(info)
            print("Replied")
        except praw.exceptions.APIException as error:
            print("Hit rate limit error.")
            with open('ids.pickle', 'wb') as handle:
                pickle.dump(ids, handle, protocol=pickle.HIGHEST_PROTOCOL)
            sleep(600)
    else:
        print("False Reply ^")

# Analyze the language of a particular sub.
def analyze(sub):
    print("Analyzing:", sub)
    subreddit = r.subreddit(sub)
    words = Counter()
    for submission in subreddit.hot(limit=300):
        comment_queue = submission.comments[:]
        while comment_queue:
            com = comment_queue.pop(0)
            if hasattr(com, 'body'):
                for word in com.body.split():
                    # Stem the word and add it to the counter.
                    word = stemmer.stem(word)
                    words[word] += 1
    languages[sub] = words
    with open('languages.pickle', 'wb') as handle:
        pickle.dump(languages, handle, protocol=pickle.HIGHEST_PROTOCOL)
    print("Analyzation complete.")

jargon(50, 10, ["test"], ml=True)
<commit_msg>Modified praw.ini<commit_after># Lior Hirschfeld
# JargonBot

# -- Imports --
import re
import pickle
import praw
import random

from datetime import datetime
from time import sleep
from define import getDefinition
from collections import Counter
from nltk.stem import *
from textblob import TextBlob
from sklearn import linear_model

# -- Setup Variables --
r = praw.Reddit('jargonBot')
stemmer = PorterStemmer()
responses = []

with open('ids.pickle', 'rb') as handle:
    try:
        ids = pickle.load(handle)
    except EOFError:
        ids = []
with open('languages.pickle', 'rb') as handle:
    languages = pickle.load(handle)
with open('count.txt', 'r') as handle:
    count = [line.split()[0] for line in handle.readlines()]
    countStemmed = [stemmer.stem(word) for word in count]

# Model Takes: [Word Popularity, Word Length, Comment Length]
# Models is a dictionary with a touple at each key containing:
# (linear regression, randomness rate)
with open('models.pickle', 'rb') as handle:
    try:
        models = pickle.load(handle)
    except EOFError:
        models = {}

# -- Methods --
def jargon(lim, rate, subs, ml=False):
    searchReddit(lim, rate, subs, ml)

# Search Reddit for words that need to be defined, and define them.
def searchReddit(lim, rate, subs, ml):
    while True:
        for sub in subs:
            searchSub(sub, lim, ml)
        with open('ids.pickle', 'wb') as handle:
            pickle.dump(ids, handle, protocol=pickle.HIGHEST_PROTOCOL)

        if ml:
            updateModels()
        sleep(rate)

# Update the machine learning model and save it.
def updateModels():
    global responses, models
    currentTime = datetime.now()
    oldResponses = [(currentTime - r["time"]).total_seconds() > 3600
                             for r in responses]
    responses = [(currentTime - r["time"]).total_seconds() < 3600
                             for r in responses]
    for r in oldResponses:
        result = 0
        url = "https://reddit.com/" + r["sID"] + "?comment=" + r["cID"]
        submission = reddit.get_submission(url=url)
        comment_queue = submission.comments[:]
        if comment_queue:
            com = comment_queue.pop(0)
            result += com.score
            comment_queue.extend(com.replies)
        while comment_queue:
            com = comment_queue.pop(0)
            text = TextBlob(com.text)
            result += text.sentiment.polarity * com.score
        models[r["sub"]][0].fit([[r["popularity"], r["wLength"], r["cLength"]]],
                              [result])

        # Update odds of random choice
        models[r]["sub"][1] *= 0.96
    with open('models.pickle', 'wb') as handle:
        pickle.dump(models, handle, protocol=pickle.HIGHEST_PROTOCOL)

# Search a sub for words that need to be defined, and define them.
def searchSub(sub, lim, ml):
    global models
    if sub not in languages:
        analyze(sub)

    subreddit = r.subreddit(sub)
    subWords = [pair[0] for pair in languages[sub].most_common(10000)]
    for submission in subreddit.hot(limit=lim):
        comment_queue = submission.comments[:]
        while comment_queue:
            com = comment_queue.pop(0)
            if not hasattr(com, 'body') or com.id in ids:
                continue
            for word in com.body.split():
                # Stem the word and check if it is rare enough to be defined.
                # Find the most similar word in count to the stemmed word.
                word = stemmer.stem(word)
                if word not in subWords:
                    for item in countStemmed:
                        if item == word:
                            word = item
                            break
                    if ml:
                        if sub not in models:
                            models[sub] = (linear_model.LinearRegression(), 1)
                            models[sub][0].fit([[1000000, 10, 10]], [10])
                        # If ML, after basic checks, predict using the model
                        # to decide whether to reply.
                        if word in count:
                            popularity = count.index(word)
                        else:
                            popularity = 1000000

                        info = {"popularity": popularity, "wLength": len(word),
                                "cLength": len(com.body), "cID": com.id,
                                "sID": submission.id, "sub": sub}

                        if popularity > 10000:
                            # Sometimes, randomly reply to train the model.
                            if random.random() < models[sub][1]:
                                reply(com, word, ml, info=info)
                            elif models[sub][0].predict([[info["popularity"],
                                    info["wLength"], info["cLength"]]]) > 0:
                                reply(com, word, ml, info=info)
                        break
                    else:
                        if word not in count[:80000]:
                            reply(com, word, ml)
                            break
            ids.append(com.id)
            comment_queue.extend(com.replies)

# Reply to a comment with a word definition.
def reply(com, word, ml, info=None):
    global responses
    print("Found Comment:" + com.id)
    reply = ""
    # Get the definition of the word (if it exists)
    result = getDefinition(word)

    if result != None:
        # A definition has been found.
        reply += """Definition of {}: {}.\n\n
                    *{}.*""".format(word.upper(), result[0].capitalize(),
                                    result[1].capitalize())
        if ml:
            reply += """
                    \n\n---------\n\n
                    I am a bot which attempts to define difficult words
                    automatically. I use machine learning to do this, and I can
                    use your feedback to improve. Feel free to leave a comment
                    to let me know what you thought of this definition!
                    """
        reply += """\n\nAll data from http://www.oed.com/. Please contact
                    /u/liortulip with any questions or concerns."""

        try:
            cID = com.reply(reply)

            if ml:
                info["time"] = datetime.now()
                info["cID"] = cID
                responses.append(info)
            print("Replied")
        except praw.exceptions.APIException as error:
            print("Hit rate limit error.")
            with open('ids.pickle', 'wb') as handle:
                pickle.dump(ids, handle, protocol=pickle.HIGHEST_PROTOCOL)
            sleep(600)
    else:
        print("False Reply ^")

# Analyze the language of a particular sub.
def analyze(sub):
    print("Analyzing:", sub)
    subreddit = r.subreddit(sub)
    words = Counter()
    for submission in subreddit.hot(limit=300):
        comment_queue = submission.comments[:]
        while comment_queue:
            com = comment_queue.pop(0)
            if hasattr(com, 'body'):
                for word in com.body.split():
                    # Stem the word and add it to the counter.
                    word = stemmer.stem(word)
                    words[word] += 1
    languages[sub] = words
    with open('languages.pickle', 'wb') as handle:
        pickle.dump(languages, handle, protocol=pickle.HIGHEST_PROTOCOL)
    print("Analyzation complete.")

jargon(50, 10, ["test"], ml=True)
<|endoftext|>


<commit_before>"""
Tests expressions that distinguish between static and non-static methods.
"""

import lldb
from lldbtest import *
import lldbutil

class CPPStaticMethodsTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    @skipUnlessDarwin
    @dsym_test
    def test_with_dsym_and_run_command(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.buildDsym()
        self.static_method_commands()

    @dwarf_test
    def test_with_dwarf_and_run_command(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.buildDwarf()
        self.static_method_commands()

    def setUp(self):
        TestBase.setUp(self)
        self.line = line_number('main.cpp', '// Break at this line')

    def static_method_commands(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.line, num_expected_locations=1, loc_exact=True)

        self.runCmd("process launch", RUN_SUCCEEDED)

        # The stop reason of the thread should be breakpoint.
        self.expect("thread list",
                    STOPPED_DUE_TO_BREAKPOINT,
                    substrs = ['stopped', 'stop reason = breakpoint'])

        self.expect("expression -- A::getStaticValue()",
                    startstr = "(int) $0 = 5")

        self.expect("expression -- my_a.getMemberValue()",
                    startstr = "(int) $1 = 3")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>XFAIL the last Windows test that calls a function in the target.<commit_after>"""
Tests expressions that distinguish between static and non-static methods.
"""

import lldb
from lldbtest import *
import lldbutil

class CPPStaticMethodsTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    @skipUnlessDarwin
    @dsym_test
    def test_with_dsym_and_run_command(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.buildDsym()
        self.static_method_commands()

    @dwarf_test
    @expectedFailureWindows
    def test_with_dwarf_and_run_command(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.buildDwarf()
        self.static_method_commands()

    def setUp(self):
        TestBase.setUp(self)
        self.line = line_number('main.cpp', '// Break at this line')

    def static_method_commands(self):
        """Test that static methods are properly distinguished from regular methods"""
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.line, num_expected_locations=1, loc_exact=True)

        self.runCmd("process launch", RUN_SUCCEEDED)

        # The stop reason of the thread should be breakpoint.
        self.expect("thread list",
                    STOPPED_DUE_TO_BREAKPOINT,
                    substrs = ['stopped', 'stop reason = breakpoint'])

        self.expect("expression -- A::getStaticValue()",
                    startstr = "(int) $0 = 5")

        self.expect("expression -- my_a.getMemberValue()",
                    startstr = "(int) $1 = 3")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2018
# - Wen Guan, wen.guan, 2018

import re

class ErrorCodes:
    """
    Pilot error codes.

    Note: Error code numbering is the same as in Pilot 1 since that is expected by the PanDA server and monitor.
    Note 2: Add error codes as they are needed in other modules. Do not import the full Pilot 1 list at once as there
    might very well be codes that can be reassigned/removed.
    """

    # Error code constants (from Pilot 1)
    GENERALERROR = 1008
    NOLOCALSPACE = 1098
    STAGEINFAILED = 1099
    REPLICANOTFOUND = 1100
    NOSUCHFILE = 1103
    USERDIRTOOLARGE = 1104
    STDOUTTOOBIG = 1106
    SETUPFAILURE = 1110
    NFSSQLITE = 1115
    QUEUEDATA = 1116
    QUEUEDATANOTOK = 1117
    OUTPUTFILETOOLARGE = 1124
    STAGEOUTFAILED = 1137
    PUTMD5MISMATCH = 1141
    CHMODTRF = 1143
    PANDAKILL = 1144
    GETMD5MISMATCH = 1145
    TRFDOWNLOADFAILURE = 1149
    LOOPINGJOB = 1150
    STAGEINTIMEOUT = 1151  # called GETTIMEOUT in Pilot 1
    STAGEOUTTIMEOUT = 1152  # called PUTTIMEOUT in Pilot 1
    NOPROXY = 1163
    MISSINGOUTPUTFILE = 1165
    SIZETOOLARGE = 1168
    GETADMISMATCH = 1171
    PUTADMISMATCH = 1172
    NOVOMSPROXY = 1177
    GETGLOBUSSYSERR = 1180
    PUTGLOBUSSYSERR = 1181
    NOSOFTWAREDIR = 1186
    NOPAYLOADMETADATA = 1187
    LFNTOOLONG = 1190
    ZEROFILESIZE = 1191
    MKDIR = 1199
    KILLSIGNAL = 1200
    SIGTERM = 1201
    SIGQUIT = 1202
    SIGSEGV = 1203
    SIGXCPU = 1204
    USERKILL = 1205  # reserved error code, currently not used by pilot
    SIGBUS = 1206
    SIGUSR1 = 1207
    MISSINGINSTALLATION = 1211
    PAYLOADOUTOFMEMORY = 1212
    REACHEDMAXTIME = 1213
    UNKNOWNPAYLOADFAILURE = 1220
    FILEEXISTS = 1221
    BADALLOC = 1223
    ESRECOVERABLE = 1224
    ESFATAL = 1228
    EXECUTEDCLONEJOB = 1234
    PAYLOADEXCEEDMAXMEM = 1235
    ESNOEVENTS = 1238
    MESSAGEHANDLINGFAILURE = 1240
    CHKSUMNOTSUP = 1242
    NORELEASEFOUND = 1244
    NOUSERTARBALL = 1246
    BADXML = 1247

    # Error code constants (new since Pilot 2)
    NOTIMPLEMENTED = 1300
    UNKNOWNEXCEPTION = 1301
    CONVERSIONFAILURE = 1302
    FILEHANDLINGFAILURE = 1303
    PAYLOADEXECUTIONFAILURE = 1305
    SINGULARITYGENERALFAILURE = 1306
    SINGULARITYNOLOOPDEVICES = 1307
    SINGULARITYBINDPOINTFAILURE = 1308
    SINGULARITYIMAGEMOUNTFAILURE = 1309
    PAYLOADEXECUTIONEXCEPTION = 1310
    NOTDEFINED = 1311
    NOTSAMELENGTH = 1312
    NOSTORAGEPROTOCOL = 1313
    UNKNOWNCHECKSUMTYPE = 1314
    UNKNOWNTRFFAILURE = 1315
    RUCIOSERVICEUNAVAILABLE = 1316
    EXCEEDEDMAXWAITTIME = 1317
    COMMUNICATIONFAILURE = 1318
    INTERNALPILOTPROBLEM = 1319
    LOGFILECREATIONFAILURE = 1320
    RUCIOLOCATIONFAILED = 1321
    RUCIOLISTREPLICASFAILED = 1322
    UNKNOWNCOPYTOOL = 1323
    SERVICENOTAVAILABLE = 1324
    SINGULARITYNOTINSTALLED = 1325
    NOREPLICAS = 1326
    UNREACHABLENETWORK = 1327
    PAYLOADSIGSEGV = 1328
    NONDETERMINISTICDDM = 1329
    JSONRETRIEVALTIMEOUT = 1330
    MISSINGINPUTFILE = 1331

    _error_messages = {
        GENERALERROR: "General pilot error, consult batch log",
        NOLOCALSPACE: "Not enough local space",
        STAGEINFAILED: "Failed to stage-in file",
        REPLICANOTFOUND: "Replica not found",
        NOSUCHFILE: "No such file or directory",
        USERDIRTOOLARGE: "User work directory too large",
        STDOUTTOOBIG: "Payload log or stdout file too big",
        SETUPFAILURE: "Failed during payload setup",
        NFSSQLITE: "NFS SQLite locking problems",
        QUEUEDATA: "Pilot could not download queuedata",
        QUEUEDATANOTOK: "Pilot found non-valid queuedata",
        OUTPUTFILETOOLARGE: "Output file too large",
        STAGEOUTFAILED: "Failed to stage-out file",
        PUTMD5MISMATCH: "md5sum mismatch on output file",
        GETMD5MISMATCH: "md5sum mismatch on input file",
        CHMODTRF: "Failed to chmod transform",
        PANDAKILL: "This job was killed by panda server",
        MISSINGOUTPUTFILE: "Local output file is missing",
        SIZETOOLARGE: "Total file size too large",
        TRFDOWNLOADFAILURE: "Transform could not be downloaded",
        LOOPINGJOB: "Looping job killed by pilot",
        STAGEINTIMEOUT: "File transfer timed out during stage-in",
        STAGEOUTTIMEOUT: "File transfer timed out during stage-out",
        NOPROXY: "Grid proxy not valid",
        GETADMISMATCH: "adler32 mismatch on input file",
        PUTADMISMATCH: "adler32 mismatch on output file",
        NOVOMSPROXY: "Voms proxy not valid",
        GETGLOBUSSYSERR: "Globus system error during stage-in",
        PUTGLOBUSSYSERR: "Globus system error during stage-out",
        NOSOFTWAREDIR: "Software directory does not exist",
        NOPAYLOADMETADATA: "Payload metadata does not exist",
        LFNTOOLONG: "LFN too long (exceeding limit of 255 characters)",
        ZEROFILESIZE: "File size cannot be zero",
        MKDIR: "Failed to create local directory",
        KILLSIGNAL: "Job terminated by unknown kill signal",
        SIGTERM: "Job killed by signal: SIGTERM",
        SIGQUIT: "Job killed by signal: SIGQUIT",
        SIGSEGV: "Job killed by signal: SIGSEGV",
        SIGXCPU: "Job killed by signal: SIGXCPU",
        SIGUSR1: "Job killed by signal: SIGUSR1",
        SIGBUS: "Job killed by signal: SIGBUS",
        USERKILL: "Job killed by user",
        MISSINGINSTALLATION: "Missing installation",
        PAYLOADOUTOFMEMORY: "Payload ran out of memory",
        REACHEDMAXTIME: "Reached batch system time limit",
        UNKNOWNPAYLOADFAILURE: "Job failed due to unknown reason (consult log file)",
        FILEEXISTS: "File already exists",
        BADALLOC: "Transform failed due to bad_alloc",
        CHKSUMNOTSUP: "Query checksum is not supported",
        NORELEASEFOUND: "No release candidates found",
        NOUSERTARBALL: "User tarball could not be downloaded from PanDA server",
        BADXML: "Badly formed XML",
        ESRECOVERABLE: "Event service: recoverable error",
        ESFATAL: "Event service: fatal error",
        EXECUTEDCLONEJOB: "Clone job is already executed",
        PAYLOADEXCEEDMAXMEM: "Payload exceeded maximum allowed memory",
        ESNOEVENTS: "Event service: no events",
        MESSAGEHANDLINGFAILURE: "Failed to handle message from payload",
        NOTIMPLEMENTED: "The class or function is not implemented",
        UNKNOWNEXCEPTION: "An unknown pilot exception has occurred",
        CONVERSIONFAILURE: "Failed to convert object data",
        FILEHANDLINGFAILURE: "Failed during file handling",
        PAYLOADEXECUTIONFAILURE: "Failed to execute payload",
        SINGULARITYGENERALFAILURE: "Singularity: general failure",
        SINGULARITYNOLOOPDEVICES: "Singularity: No more available loop devices",
        SINGULARITYBINDPOINTFAILURE: "Singularity: Not mounting requested bind point",
        SINGULARITYIMAGEMOUNTFAILURE: "Singularity: Failed to mount image",
        SINGULARITYNOTINSTALLED: "Singularity: not installed",
        PAYLOADEXECUTIONEXCEPTION: "Exception caught during payload execution",
        NOTDEFINED: "Not defined",
        NOTSAMELENGTH: "Not same length",
        NOSTORAGEPROTOCOL: "No protocol defined for storage endpoint",
        UNKNOWNCHECKSUMTYPE: "Unknown checksum type",
        UNKNOWNTRFFAILURE: "Unknown transform failure",
        RUCIOSERVICEUNAVAILABLE: "Rucio: Service unavailable",
        EXCEEDEDMAXWAITTIME: "Exceeded maximum waiting time",
        COMMUNICATIONFAILURE: "Failed to communicate with server",
        INTERNALPILOTPROBLEM: "An internal Pilot problem has occurred (consult Pilot log)",
        LOGFILECREATIONFAILURE: "Failed during creation of log file",
        RUCIOLOCATIONFAILED: "Failed to get client location for Rucio",
        RUCIOLISTREPLICASFAILED: "Failed to get replicas from Rucio",
        UNKNOWNCOPYTOOL: "Unknown copy tool",
        SERVICENOTAVAILABLE: "Service not available at the moment",
        NOREPLICAS: "No matching replicas were found in list_replicas() output",
        UNREACHABLENETWORK: "Unable to stage-in file since network is unreachable",
        PAYLOADSIGSEGV: "SIGSEGV: Invalid memory reference or a segmentation fault",
        NONDETERMINISTICDDM: "Failed to construct SURL for non-deterministic ddm (update AGIS)",
        JSONRETRIEVALTIMEOUT: "JSON retrieval timed out",
        MISSINGINPUTFILE: "Input file is missing in storage element"
    }

    put_error_codes = [1135, 1136, 1137, 1141, 1152, 1181]
    recoverable_error_codes = [0] + put_error_codes

    def get_kill_signal_error_code(self, signal):
        """
        Match a kill signal with a corresponding Pilot error code.

        :param signal: signal name (string).
        :return: Pilot error code (integer).
        """

        signals_dictionary = {'SIGTERM': self.SIGTERM,
                              'SIGQUIT': self.SIGQUIT,
                              'SIGSEGV': self.SIGSEGV,
                              'SIGXCPU': self.SIGXCPU,
                              'SIGUSR1': self.SIGUSR1,
                              'SIGBUS': self.SIGBUS}

        return signals_dictionary.get(signal, self.KILLSIGNAL)

    def get_error_message(self, errorcode):
        """
        Return the error message corresponding to the given error code.

        :param errorcode:
        :return: errormessage (string)
        """

        if errorcode in self._error_messages:
            return self._error_messages[errorcode]
        else:
            return "Unknown error code: %d" % errorcode

    def add_error_code(self, errorcode, pilot_error_codes=[], pilot_error_diags=[]):
        """
        Add pilot error code to list of error codes.
        This function adds the given error code to the list of all errors that have occurred. This is needed since
        several errors can happen; e.g. a stage-in error can be followed by a stage-out error during the log transfer.
        The full list of errors is dumped to the log, but only the first error is reported to the server.
        The function also sets the corresponding error message.

        :param errorcode: pilot error code (integer)
        :param pilot_error_codes: list of pilot error codes (list of integers)
        :param pilot_error_diags: list of pilot error diags (list of strings)
        :return: pilotErrorCodes, pilotErrorDiags
        """

        # do nothing if the error code has already been added
        if errorcode not in pilot_error_codes:
            pilot_error_codes.append(errorcode)
            pilot_error_diags.append(self.get_error_message(errorcode))

        return pilot_error_codes, pilot_error_diags

    def report_errors(self, pilot_error_codes, pilot_error_diags):
        """
        Report all errors that occurred during running.
        The function should be called towards the end of running a job.

        :param pilot_error_codes: list of pilot error codes (list of integers)
        :param pilot_error_diags: list of pilot error diags (list of strings)
        :return: error_report (string)
        """

        i = 0
        if pilot_error_codes == []:
            report = "no pilot errors were reported"
        else:
            report = "Nr.\tError code\tError diagnostics"
            for errorcode in pilot_error_codes:
                i += 1
                report += "\n%d.\t%d\t%s" % (i, errorcode, pilot_error_diags[i - 1])

        return report

    def resolve_transform_error(self, exit_code, stderr):
        """
        Assign a pilot error code to a specific transform error.
        :param exit_code: transform exit code.
        :param stderr: transform stderr
        :return: pilot error code (int)
        """

        if exit_code == 251 and "Not mounting requested bind point" in stderr:
            ec = self.SINGULARITYBINDPOINTFAILURE
        elif exit_code == 255 and "No more available loop devices" in stderr:
            ec = self.SINGULARITYNOLOOPDEVICES
        elif exit_code == 255 and "Failed to mount image" in stderr:
            ec = self.SINGULARITYIMAGEMOUNTFAILURE
        elif exit_code == 255 and "Operation not permitted" in stderr:
            ec = self.SINGULARITYGENERALFAILURE
        elif exit_code == 64 and "Singularity is not installed" in stderr:
            ec = self.SINGULARITYNOTINSTALLED
        elif exit_code == -1:
            ec = self.UNKNOWNTRFFAILURE
        else:
            # do not assign a pilot error code for unidentified transform error, return 0
            ec = 0

        return ec

    def extract_stderr_msg(self, stderr):
        """
        Extract the ERROR or WARNING message from the singularity stderr.
        :param stderr: string.
        :return: string.
        """

        msg = ""
        pattern = r"ERROR +\: (.+)"
        found = re.findall(pattern, stderr)
        if len(found) > 0:
            msg = found[0]
        else:
            pattern = r"WARNING\: (.+)"
            found = re.findall(pattern, stderr)
            if len(found) > 0:
                msg = found[0]

        return msg

    @classmethod
    def is_recoverable(self, code=0):
        """
        Determine whether code is a recoverable error code or not.

        :param code: Pilot error code (int).
        :return: boolean.
        """

        return code in self.recoverable_error_codes
<commit_msg>Added new error code 1332, BLACKHOLE: "Black hole detected in file system (consult Pilot log)"<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2018
# - Wen Guan, wen.guan, 2018

import re

class ErrorCodes:
    """
    Pilot error codes.

    Note: Error code numbering is the same as in Pilot 1 since that is expected by the PanDA server and monitor.
    Note 2: Add error codes as they are needed in other modules. Do not import the full Pilot 1 list at once as there
    might very well be codes that can be reassigned/removed.
    """

    # Error code constants (from Pilot 1)
    GENERALERROR = 1008
    NOLOCALSPACE = 1098
    STAGEINFAILED = 1099
    REPLICANOTFOUND = 1100
    NOSUCHFILE = 1103
    USERDIRTOOLARGE = 1104
    STDOUTTOOBIG = 1106
    SETUPFAILURE = 1110
    NFSSQLITE = 1115
    QUEUEDATA = 1116
    QUEUEDATANOTOK = 1117
    OUTPUTFILETOOLARGE = 1124
    STAGEOUTFAILED = 1137
    PUTMD5MISMATCH = 1141
    CHMODTRF = 1143
    PANDAKILL = 1144
    GETMD5MISMATCH = 1145
    TRFDOWNLOADFAILURE = 1149
    LOOPINGJOB = 1150
    STAGEINTIMEOUT = 1151  # called GETTIMEOUT in Pilot 1
    STAGEOUTTIMEOUT = 1152  # called PUTTIMEOUT in Pilot 1
    NOPROXY = 1163
    MISSINGOUTPUTFILE = 1165
    SIZETOOLARGE = 1168
    GETADMISMATCH = 1171
    PUTADMISMATCH = 1172
    NOVOMSPROXY = 1177
    GETGLOBUSSYSERR = 1180
    PUTGLOBUSSYSERR = 1181
    NOSOFTWAREDIR = 1186
    NOPAYLOADMETADATA = 1187
    LFNTOOLONG = 1190
    ZEROFILESIZE = 1191
    MKDIR = 1199
    KILLSIGNAL = 1200
    SIGTERM = 1201
    SIGQUIT = 1202
    SIGSEGV = 1203
    SIGXCPU = 1204
    USERKILL = 1205  # reserved error code, currently not used by pilot
    SIGBUS = 1206
    SIGUSR1 = 1207
    MISSINGINSTALLATION = 1211
    PAYLOADOUTOFMEMORY = 1212
    REACHEDMAXTIME = 1213
    UNKNOWNPAYLOADFAILURE = 1220
    FILEEXISTS = 1221
    BADALLOC = 1223
    ESRECOVERABLE = 1224
    ESFATAL = 1228
    EXECUTEDCLONEJOB = 1234
    PAYLOADEXCEEDMAXMEM = 1235
    ESNOEVENTS = 1238
    MESSAGEHANDLINGFAILURE = 1240
    CHKSUMNOTSUP = 1242
    NORELEASEFOUND = 1244
    NOUSERTARBALL = 1246
    BADXML = 1247

    # Error code constants (new since Pilot 2)
    NOTIMPLEMENTED = 1300
    UNKNOWNEXCEPTION = 1301
    CONVERSIONFAILURE = 1302
    FILEHANDLINGFAILURE = 1303
    PAYLOADEXECUTIONFAILURE = 1305
    SINGULARITYGENERALFAILURE = 1306
    SINGULARITYNOLOOPDEVICES = 1307
    SINGULARITYBINDPOINTFAILURE = 1308
    SINGULARITYIMAGEMOUNTFAILURE = 1309
    PAYLOADEXECUTIONEXCEPTION = 1310
    NOTDEFINED = 1311
    NOTSAMELENGTH = 1312
    NOSTORAGEPROTOCOL = 1313
    UNKNOWNCHECKSUMTYPE = 1314
    UNKNOWNTRFFAILURE = 1315
    RUCIOSERVICEUNAVAILABLE = 1316
    EXCEEDEDMAXWAITTIME = 1317
    COMMUNICATIONFAILURE = 1318
    INTERNALPILOTPROBLEM = 1319
    LOGFILECREATIONFAILURE = 1320
    RUCIOLOCATIONFAILED = 1321
    RUCIOLISTREPLICASFAILED = 1322
    UNKNOWNCOPYTOOL = 1323
    SERVICENOTAVAILABLE = 1324
    SINGULARITYNOTINSTALLED = 1325
    NOREPLICAS = 1326
    UNREACHABLENETWORK = 1327
    PAYLOADSIGSEGV = 1328
    NONDETERMINISTICDDM = 1329
    JSONRETRIEVALTIMEOUT = 1330
    MISSINGINPUTFILE = 1331
    BLACKHOLE = 1332

    _error_messages = {
        GENERALERROR: "General pilot error, consult batch log",
        NOLOCALSPACE: "Not enough local space",
        STAGEINFAILED: "Failed to stage-in file",
        REPLICANOTFOUND: "Replica not found",
        NOSUCHFILE: "No such file or directory",
        USERDIRTOOLARGE: "User work directory too large",
        STDOUTTOOBIG: "Payload log or stdout file too big",
        SETUPFAILURE: "Failed during payload setup",
        NFSSQLITE: "NFS SQLite locking problems",
        QUEUEDATA: "Pilot could not download queuedata",
        QUEUEDATANOTOK: "Pilot found non-valid queuedata",
        OUTPUTFILETOOLARGE: "Output file too large",
        STAGEOUTFAILED: "Failed to stage-out file",
        PUTMD5MISMATCH: "md5sum mismatch on output file",
        GETMD5MISMATCH: "md5sum mismatch on input file",
        CHMODTRF: "Failed to chmod transform",
        PANDAKILL: "This job was killed by panda server",
        MISSINGOUTPUTFILE: "Local output file is missing",
        SIZETOOLARGE: "Total file size too large",
        TRFDOWNLOADFAILURE: "Transform could not be downloaded",
        LOOPINGJOB: "Looping job killed by pilot",
        STAGEINTIMEOUT: "File transfer timed out during stage-in",
        STAGEOUTTIMEOUT: "File transfer timed out during stage-out",
        NOPROXY: "Grid proxy not valid",
        GETADMISMATCH: "adler32 mismatch on input file",
        PUTADMISMATCH: "adler32 mismatch on output file",
        NOVOMSPROXY: "Voms proxy not valid",
        GETGLOBUSSYSERR: "Globus system error during stage-in",
        PUTGLOBUSSYSERR: "Globus system error during stage-out",
        NOSOFTWAREDIR: "Software directory does not exist",
        NOPAYLOADMETADATA: "Payload metadata does not exist",
        LFNTOOLONG: "LFN too long (exceeding limit of 255 characters)",
        ZEROFILESIZE: "File size cannot be zero",
        MKDIR: "Failed to create local directory",
        KILLSIGNAL: "Job terminated by unknown kill signal",
        SIGTERM: "Job killed by signal: SIGTERM",
        SIGQUIT: "Job killed by signal: SIGQUIT",
        SIGSEGV: "Job killed by signal: SIGSEGV",
        SIGXCPU: "Job killed by signal: SIGXCPU",
        SIGUSR1: "Job killed by signal: SIGUSR1",
        SIGBUS: "Job killed by signal: SIGBUS",
        USERKILL: "Job killed by user",
        MISSINGINSTALLATION: "Missing installation",
        PAYLOADOUTOFMEMORY: "Payload ran out of memory",
        REACHEDMAXTIME: "Reached batch system time limit",
        UNKNOWNPAYLOADFAILURE: "Job failed due to unknown reason (consult log file)",
        FILEEXISTS: "File already exists",
        BADALLOC: "Transform failed due to bad_alloc",
        CHKSUMNOTSUP: "Query checksum is not supported",
        NORELEASEFOUND: "No release candidates found",
        NOUSERTARBALL: "User tarball could not be downloaded from PanDA server",
        BADXML: "Badly formed XML",
        ESRECOVERABLE: "Event service: recoverable error",
        ESFATAL: "Event service: fatal error",
        EXECUTEDCLONEJOB: "Clone job is already executed",
        PAYLOADEXCEEDMAXMEM: "Payload exceeded maximum allowed memory",
        ESNOEVENTS: "Event service: no events",
        MESSAGEHANDLINGFAILURE: "Failed to handle message from payload",
        NOTIMPLEMENTED: "The class or function is not implemented",
        UNKNOWNEXCEPTION: "An unknown pilot exception has occurred",
        CONVERSIONFAILURE: "Failed to convert object data",
        FILEHANDLINGFAILURE: "Failed during file handling",
        PAYLOADEXECUTIONFAILURE: "Failed to execute payload",
        SINGULARITYGENERALFAILURE: "Singularity: general failure",
        SINGULARITYNOLOOPDEVICES: "Singularity: No more available loop devices",
        SINGULARITYBINDPOINTFAILURE: "Singularity: Not mounting requested bind point",
        SINGULARITYIMAGEMOUNTFAILURE: "Singularity: Failed to mount image",
        SINGULARITYNOTINSTALLED: "Singularity: not installed",
        PAYLOADEXECUTIONEXCEPTION: "Exception caught during payload execution",
        NOTDEFINED: "Not defined",
        NOTSAMELENGTH: "Not same length",
        NOSTORAGEPROTOCOL: "No protocol defined for storage endpoint",
        UNKNOWNCHECKSUMTYPE: "Unknown checksum type",
        UNKNOWNTRFFAILURE: "Unknown transform failure",
        RUCIOSERVICEUNAVAILABLE: "Rucio: Service unavailable",
        EXCEEDEDMAXWAITTIME: "Exceeded maximum waiting time",
        COMMUNICATIONFAILURE: "Failed to communicate with server",
        INTERNALPILOTPROBLEM: "An internal Pilot problem has occurred (consult Pilot log)",
        LOGFILECREATIONFAILURE: "Failed during creation of log file",
        RUCIOLOCATIONFAILED: "Failed to get client location for Rucio",
        RUCIOLISTREPLICASFAILED: "Failed to get replicas from Rucio",
        UNKNOWNCOPYTOOL: "Unknown copy tool",
        SERVICENOTAVAILABLE: "Service not available at the moment",
        NOREPLICAS: "No matching replicas were found in list_replicas() output",
        UNREACHABLENETWORK: "Unable to stage-in file since network is unreachable",
        PAYLOADSIGSEGV: "SIGSEGV: Invalid memory reference or a segmentation fault",
        NONDETERMINISTICDDM: "Failed to construct SURL for non-deterministic ddm (update AGIS)",
        JSONRETRIEVALTIMEOUT: "JSON retrieval timed out",
        MISSINGINPUTFILE: "Input file is missing in storage element",
        BLACKHOLE: "Black hole detected in file system (consult Pilot log)"
    }

    put_error_codes = [1135, 1136, 1137, 1141, 1152, 1181]
    recoverable_error_codes = [0] + put_error_codes

    def get_kill_signal_error_code(self, signal):
        """
        Match a kill signal with a corresponding Pilot error code.

        :param signal: signal name (string).
        :return: Pilot error code (integer).
        """

        signals_dictionary = {'SIGTERM': self.SIGTERM,
                              'SIGQUIT': self.SIGQUIT,
                              'SIGSEGV': self.SIGSEGV,
                              'SIGXCPU': self.SIGXCPU,
                              'SIGUSR1': self.SIGUSR1,
                              'SIGBUS': self.SIGBUS}

        return signals_dictionary.get(signal, self.KILLSIGNAL)

    def get_error_message(self, errorcode):
        """
        Return the error message corresponding to the given error code.

        :param errorcode:
        :return: errormessage (string)
        """

        if errorcode in self._error_messages:
            return self._error_messages[errorcode]
        else:
            return "Unknown error code: %d" % errorcode

    def add_error_code(self, errorcode, pilot_error_codes=[], pilot_error_diags=[]):
        """
        Add pilot error code to list of error codes.
        This function adds the given error code to the list of all errors that have occurred. This is needed since
        several errors can happen; e.g. a stage-in error can be followed by a stage-out error during the log transfer.
        The full list of errors is dumped to the log, but only the first error is reported to the server.
        The function also sets the corresponding error message.

        :param errorcode: pilot error code (integer)
        :param pilot_error_codes: list of pilot error codes (list of integers)
        :param pilot_error_diags: list of pilot error diags (list of strings)
        :return: pilotErrorCodes, pilotErrorDiags
        """

        # do nothing if the error code has already been added
        if errorcode not in pilot_error_codes:
            pilot_error_codes.append(errorcode)
            pilot_error_diags.append(self.get_error_message(errorcode))

        return pilot_error_codes, pilot_error_diags

    def report_errors(self, pilot_error_codes, pilot_error_diags):
        """
        Report all errors that occurred during running.
        The function should be called towards the end of running a job.

        :param pilot_error_codes: list of pilot error codes (list of integers)
        :param pilot_error_diags: list of pilot error diags (list of strings)
        :return: error_report (string)
        """

        i = 0
        if pilot_error_codes == []:
            report = "no pilot errors were reported"
        else:
            report = "Nr.\tError code\tError diagnostics"
            for errorcode in pilot_error_codes:
                i += 1
                report += "\n%d.\t%d\t%s" % (i, errorcode, pilot_error_diags[i - 1])

        return report

    def resolve_transform_error(self, exit_code, stderr):
        """
        Assign a pilot error code to a specific transform error.
        :param exit_code: transform exit code.
        :param stderr: transform stderr
        :return: pilot error code (int)
        """

        if exit_code == 251 and "Not mounting requested bind point" in stderr:
            ec = self.SINGULARITYBINDPOINTFAILURE
        elif exit_code == 255 and "No more available loop devices" in stderr:
            ec = self.SINGULARITYNOLOOPDEVICES
        elif exit_code == 255 and "Failed to mount image" in stderr:
            ec = self.SINGULARITYIMAGEMOUNTFAILURE
        elif exit_code == 255 and "Operation not permitted" in stderr:
            ec = self.SINGULARITYGENERALFAILURE
        elif exit_code == 64 and "Singularity is not installed" in stderr:
            ec = self.SINGULARITYNOTINSTALLED
        elif exit_code == -1:
            ec = self.UNKNOWNTRFFAILURE
        else:
            # do not assign a pilot error code for unidentified transform error, return 0
            ec = 0

        return ec

    def extract_stderr_msg(self, stderr):
        """
        Extract the ERROR or WARNING message from the singularity stderr.
        :param stderr: string.
        :return: string.
        """

        msg = ""
        pattern = r"ERROR +\: (.+)"
        found = re.findall(pattern, stderr)
        if len(found) > 0:
            msg = found[0]
        else:
            pattern = r"WARNING\: (.+)"
            found = re.findall(pattern, stderr)
            if len(found) > 0:
                msg = found[0]

        return msg

    @classmethod
    def is_recoverable(self, code=0):
        """
        Determine whether code is a recoverable error code or not.

        :param code: Pilot error code (int).
        :return: boolean.
        """

        return code in self.recoverable_error_codes
<|endoftext|>


<commit_before>#coding=utf-8
import sys
import os

from flask import Flask, g, request, render_template, \
                    redirect, url_for, jsonify, _app_ctx_stack
from flask.ext import login
from flask.ext.mongoengine import MongoEngine
from social.apps.flask_app.routes import social_auth
from social.apps.flask_app.template_filters import backends
from social.apps.flask_app.models import User

#Your setting file
import settings

def static(filename):
    filepath = os.path.join(os.path.dirname(__file__), 'static', filename)
    last_modification = '%d' % os.path.getmtime(filepath)
    return url_for('static', filename=filename) + '?' + last_modification

def create_app():
    app = Flask(__name__)
    app.config.from_object(settings)
    @app.context_processor
    def inject_static():
        return dict(static=static)
    return app

app = create_app()
db = MongoEngine()
db.init_app(app)

app.register_blueprint(social_auth)

login_manager = login.LoginManager()
login_manager.login_view = 'main'
login_manager.login_message = ''
login_manager.setup_app(app)

from flask_reveal import routes

@app.template_filter('bool')
def bool_filter(s):
    return str(s).lower()

@login_manager.user_loader
def load_user(userid):
    try:
        return User.objects(id=userid).first()
    except (TypeError, ValueError):
        pass

@app.before_request
def global_user():
    g.user = login.current_user

@app.context_processor
def inject_user():
    try:
        return {'user': g.user}
    except AttributeError:
        return {'user': None}
app.jinja_env.filters['bool'] = bool_filter
app.context_processor(backends)
<commit_msg>modify seetings dir<commit_after>#coding=utf-8
import sys
import os

from flask import Flask, g, request, render_template, \
                    redirect, url_for, jsonify, _app_ctx_stack
from flask.ext import login
from flask.ext.mongoengine import MongoEngine
from social.apps.flask_app.routes import social_auth
from social.apps.flask_app.template_filters import backends
from social.apps.flask_app.models import User

#Your setting file
from flask_reveal import settings

def static(filename):
    filepath = os.path.join(os.path.dirname(__file__), 'static', filename)
    last_modification = '%d' % os.path.getmtime(filepath)
    return url_for('static', filename=filename) + '?' + last_modification

def create_app():
    app = Flask(__name__)
    app.config.from_object(settings)
    @app.context_processor
    def inject_static():
        return dict(static=static)
    return app

app = create_app()
db = MongoEngine()
db.init_app(app)

app.register_blueprint(social_auth)

login_manager = login.LoginManager()
login_manager.login_view = 'main'
login_manager.login_message = ''
login_manager.setup_app(app)

from flask_reveal import routes

@app.template_filter('bool')
def bool_filter(s):
    return str(s).lower()

@login_manager.user_loader
def load_user(userid):
    try:
        return User.objects(id=userid).first()
    except (TypeError, ValueError):
        pass

@app.before_request
def global_user():
    g.user = login.current_user

@app.context_processor
def inject_user():
    try:
        return {'user': g.user}
    except AttributeError:
        return {'user': None}
app.jinja_env.filters['bool'] = bool_filter
app.context_processor(backends)
<|endoftext|>


<commit_before>##############################################################################
#
#    Copyright (C) 2020 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Quentin Gigon, Emanuel Cino
#
#    The licence is in the file __manifest__.py
#
##############################################################################
from odoo.http import request, route, Controller, local_redirect
from odoo.addons.cms_form.controllers.main import FormControllerMixin

class ProjectsController(Controller, FormControllerMixin):
    @route("/projects", auth="public", website=True)
    def get_projects_list(self, **kwargs):
        project_obj = request.env["crowdfunding.project"]

        # TODO connect pagination to backend -> CO-3213
        return request.render(
            "crowdfunding_compassion.project_list_page",
            {"projects": project_obj.sudo().get_active_projects()},
        )

    @route(['/projects/create', '/projects/create/page/<int:page>'],
           auth="public",
           type="http",
           method='POST',
           website=True)
    def project_creation(self, page=1, **kwargs):
        values = kwargs.copy()
        values["form_model_key"] = "cms.form.crowdfunding.project.step" + str(page)
        values.update({
            "is_published": False,
            "page": page
        })
        # This allows the translation to still work on the page
        project_creation_form = self.get_form("crowdfunding.project", **values)
        project_creation_form.form_process()
        values.update({
            "user": request.env.user,
            "form": project_creation_form}
        )
        project_creation_form = values["form"]
        if project_creation_form.form_success:
            # Force saving session, otherwise we lose values between steps
            request.session.save_request_data()
            return local_redirect(project_creation_form.form_next_url())
        else:
            return request.render(
                "crowdfunding_compassion.project_creation_view_template", values,
                headers={'Cache-Control': 'no-cache'}
            )

    @route("/projects/create/confirm/",
           auth="public",
           type="http",
           method='POST',
           website=True)
    def project_validation(self, **kwargs):
        return request.render(
            "crowdfunding_compassion.project_creation_confirmation_view_template",
            **kwargs
        )
<commit_msg>Fix access right of project list page<commit_after>##############################################################################
#
#    Copyright (C) 2020 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Quentin Gigon, Emanuel Cino
#
#    The licence is in the file __manifest__.py
#
##############################################################################
from odoo.http import request, route, Controller, local_redirect
from odoo.addons.cms_form.controllers.main import FormControllerMixin

class ProjectsController(Controller, FormControllerMixin):
    @route("/projects", auth="public", website=True)
    def get_projects_list(self, **kwargs):
        project_obj = request.env["crowdfunding.project"].sudo()

        # TODO connect pagination to backend -> CO-3213
        return request.render(
            "crowdfunding_compassion.project_list_page",
            {"projects": project_obj.sudo().get_active_projects()},
        )

    @route(['/projects/create', '/projects/create/page/<int:page>'],
           auth="public",
           type="http",
           method='POST',
           website=True)
    def project_creation(self, page=1, **kwargs):
        values = kwargs.copy()
        values["form_model_key"] = "cms.form.crowdfunding.project.step" + str(page)
        values.update({
            "is_published": False,
            "page": page
        })
        # This allows the translation to still work on the page
        project_creation_form = self.get_form("crowdfunding.project", **values)
        project_creation_form.form_process()
        values.update({
            "user": request.env.user,
            "form": project_creation_form}
        )
        project_creation_form = values["form"]
        if project_creation_form.form_success:
            # Force saving session, otherwise we lose values between steps
            request.session.save_request_data()
            return local_redirect(project_creation_form.form_next_url())
        else:
            return request.render(
                "crowdfunding_compassion.project_creation_view_template", values,
                headers={'Cache-Control': 'no-cache'}
            )

    @route("/projects/create/confirm/",
           auth="public",
           type="http",
           method='POST',
           website=True)
    def project_validation(self, **kwargs):
        return request.render(
            "crowdfunding_compassion.project_creation_confirmation_view_template",
            **kwargs
        )
<|endoftext|>


<commit_before>'''
conatcts.py
----------
Contact Book Application to CRUD personal contacts via a shelve DB
    - Functions
    - Objects
    - Modules
----------
Python Demo Workshop, March 22nd 2017
'''
### Import Modules
import shelve # Database Management Module [https://docs.python.org/2/library/shelve.html]
import os

# Database Name
DB_NAME = '.Contacts.db'

### Contact Class
# Python Class definiton of model used to represent personal contacts
class Contact():

    # Constructor Method
    def __init__(self, name, phone, address, email, nickname):
        self.name = name
        self.phone = phone
        self.address = address
        self.email = email
        self.nickname = nickname

# open_DB
# Returns open instance of shelve object
def open_DB():
    try:
        db = shelve.open(DB_NAME)

        # Syntax to check if db was opened properly
        # Verbose condition needed, as first time DB is opened it is empty
        #   -> Checking <if db> would not display confirmation, even though its true
        if db is not None:
            print('Opened DB to read: {0}'.format(DB_NAME))

        return db

    # ERROR CASE[1]: Unable to open Database
    except Exception:
        print('ERROR[1]: Unable to open Database')
        print('Exception information: {0}'.format(e))

        exit(1) # !! IF WE GOT HERE WE MESSED UP !! *#

# close_DB
# Closes instance of shelve object
def close_DB(db):
    try:
        db.close()
        return True

    # ERROR CASE[1]: Unable to close Database
    except Exception:
        print('ERROR[2]: Unable to open Database')
        print('Exception information: {0}'.format(e))

        exit(2) # !! IF WE GOT HERE WE MESSED UP !! *#

# store_contact
# CREATE
# Creates and saves new Contact object to DB
def store_contact(db):
    print('You are going to add a contact to your book')

    # Get new contact information
    name = raw_input('Please enter the persons name: ')
    phone = raw_input('Please enter the telephone number: ')
    address = raw_input('Please enter the address: ')
    email = raw_input('Please enter the email: ')
    nick = raw_input('Please enter the nickname (or leave blank for none): ')

    # Account for no nickname
    if nick == '':
        nick = None

    # Create new Contact Instance
    contact = Contact(name, phone, address, email, nick)

    # Save to DB
    db[phone] = contact # Store contact as value with key being phone number

    return True

# print_DB
# READ
# Prints all currently stored in DB
def print_DB(db):
    print('Printing database...')

    # Loop through dictionary to print values
    for value in db.values():
        print('Name: {}'.format(value.name))
        print('Phone: {}'.format(value.phone))
        print('Address: {}'.format(value.address))
        print('Email: {}'.format(value.email))

        if value.nickname:
            print('Nickname: {}'.format(value.nickname))

        print '-' * 75

# edit_contact
# UPDATE
# Edits a contact accessed through key [phone number]
def edit_contact(db):
    pass

#delet_contact
# DELETE
# Deletes a particular contact from the DB
def delete_contact(db):
    pass

# main
# Driver of program; Calls other functions that do stuff
def main():
    db = open_DB()

    # Driver to go here

    close_DB(db)

# Entry Point of Script
if __name__ == '__main__':
    main()
<commit_msg>Added edit_contact<commit_after>'''
conatcts.py
----------
Contact Book Application to CRUD personal contacts via a shelve DB
    - Functions
    - Objects
    - Modules
----------
Python Demo Workshop, March 22nd 2017
'''
### Import Modules
import shelve # Database Management Module [https://docs.python.org/2/library/shelve.html]
import os

# Database Name
DB_NAME = '.Contacts.db'

### Contact Class
# Python Class definiton of model used to represent personal contacts
class Contact():

    # Constructor Method
    def __init__(self, name, phone, address, email, nickname):
        self.name = name
        self.phone = phone
        self.address = address
        self.email = email
        self.nickname = nickname

# open_DB
# Returns open instance of shelve object
def open_DB():
    try:
        db = shelve.open(DB_NAME)

        # Syntax to check if db was opened properly
        # Verbose condition needed, as first time DB is opened it is empty
        #   -> Checking <if db> would not display confirmation, even though its true
        if db is not None:
            print('Opened DB to read: {0}'.format(DB_NAME))

        return db

    # ERROR CASE[1]: Unable to open Database
    except Exception:
        print('ERROR[1]: Unable to open Database')
        print('Exception information: {0}'.format(e))

        exit(1) # !! IF WE GOT HERE WE MESSED UP !! *#

# close_DB
# Closes instance of shelve object
def close_DB(db):
    try:
        db.close()
        return True

    # ERROR CASE[1]: Unable to close Database
    except Exception:
        print('ERROR[2]: Unable to open Database')
        print('Exception information: {0}'.format(e))

        exit(2) # !! IF WE GOT HERE WE MESSED UP !! *#

# store_contact
# CREATE
# Creates and saves new Contact object to DB
def store_contact(db):
    print('You are going to add a contact to your book...')

    # Get new contact information
    name = raw_input('Please enter the persons name: ')
    phone = raw_input('Please enter the telephone number: ')
    address = raw_input('Please enter the address: ')
    email = raw_input('Please enter the email: ')
    nick = raw_input('Please enter the nickname (or leave blank for none): ')

    # Account for no nickname
    if nick == '':
        nick = None

    # Create new Contact Instance
    contact = Contact(name, phone, address, email, nick)

    # Save to DB
    db[phone] = contact # Store contact as value with key being phone number

    return True

# print_DB
# READ
# Prints all currently stored in DB
def print_DB(db):
    print('Printing database...')

    # Loop through dictionary to print values
    for value in db.values():
        print('Name: {}'.format(value.name))
        print('Phone: {}'.format(value.phone))
        print('Address: {}'.format(value.address))
        print('Email: {}'.format(value.email))

        if value.nickname:
            print('Nickname: {}'.format(value.nickname))

        print '-' * 75

# edit_contact
# UPDATE
# Edits a contact accessed through key [phone number]
def edit_contact(db):
    print('Going to update a record in the database...')

    # Print out key/value pairs
    # iteritems() will return a list of tuples that contain the key in the first position
    # and value in the second
    for key, value in db.iteritems():
        print('Contact name: {}'.format(value.name))
        print('Contact phone number: {}'.format(key))
        print '-' * 75

    idx = raw_input('Please enter the phone number of the contact to edit: ')

    try:
        contact = db.get(idx)
        print('Going to update {}...'.format(contact.name))

        # Phone number is not reassigned due to its use as primary key
        # -> Is there a way around this?
        contact.name = raw_input('Please enter new name of contact: ')
        contact.address = raw_input('Please enter the new address: ')
        contact.email = raw_input('Please enter the new email: ')
        contact.nickname = raw_input('Please enter the new nickname (or leave blank for none): ')

        if contact.nickname == '':
            contact.nickname = None

        db[idx] = contact

        return True

    # ERROR CASE[3]: Contact with matching key not found in DB
    except KeyError:
        print('No contact found with phone number: {}'.format(idx))
        print('Exiting update function...')

        return False

#delet_contact
# DELETE
# Deletes a particular contact from the DB
def delete_contact(db):
    pass

# main
# Driver of program; Calls other functions that do stuff
def main():
    db = open_DB()

    # Driver to go here

    isClosed = close_DB(db)

    if isClosed:
        print('Closed {} OK'.format(DB_NAME))

# Entry Point of Script
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Contains code that gets run inside our mako template
Debugging python-in-mako is terrible, so we've moved the actual code out to its own file
"""
from django.conf import settings

def load_sfe_i18n_messages(locale):
    """
    Loads i18n data from studio-frontend's published files.
    """
    messages = "{}"

    if locale != 'en':
        # because en is the default, studio-frontend will have it loaded by default
        messages_path = "{base}/studio-frontend/dist/i18n/messages/{locale}.json".format(
            base=settings.STATIC_ROOT_BASE,
            locale=locale.replace('-', '_')  # files from Transifex use _, platform follows RFC-5646 and uses -
        )
        with open(messages_path) as inputfile:
            messages = inputfile.read()

    return messages
<commit_msg>make use of to_locale in studiofrontend templating<commit_after>"""
Contains code that gets run inside our mako template
Debugging python-in-mako is terrible, so we've moved the actual code out to its own file
"""
from django.conf import settings
from django.utils.translation import to_locale

def load_sfe_i18n_messages(language):
    """
    Loads i18n data from studio-frontend's published files.
    """
    messages = "{}"

    if language != 'en':
        # because en is the default, studio-frontend will have it loaded by default
        messages_path = "{base}/studio-frontend/dist/i18n/messages/{locale}.json".format(
            base=settings.STATIC_ROOT_BASE,
            locale=to_locale(language)
        )
        with open(messages_path) as inputfile:
            messages = inputfile.read()

    return messages
<|endoftext|>


<commit_before>import sys
from collections import defaultdict
from functools import wraps
import copy
import platform
import pkg_resources

import gringo

import equibel as eb

if platform.system() == 'Linux':
    if platform.architecture()[0] == '64bit':
        if platform.dist()[0] in ['centos', 'CentOS']:
            import equibel.includes.linux.bit64.centos.gringo as gringo
            from equibel.includes.linux.bit64.centos.gringo import Control, Model, Fun
        else:
            import equibel.includes.linux.bit64.gringo as gringo
            from equibel.includes.linux.bit64.gringo import Control, Model, Fun
    elif platform.architecture()[0] == '32bit':
        import equibel.includes.linux.bit32.gringo as gringo
        from equibel.includes.linux.bit32.gringo import Control, Model, Fun
elif platform.system() == 'Darwin':
    import equibel.includes.mac.gringo as gringo
    from equibel.includes.mac.gringo import Control, Model, Fun

import equibel.FormulaExtractor as FormulaExtractor
import equibel.formatters.ASP_Formatter as ASP_Formatter
#from equibel.simbool.simplify import simplify

EQ_SETS_FILE = pkg_resources.resource_filename('equibel', 'asp/eq_sets.lp')

CONTAINMENT = 0
CARDINALITY = 1

def completion(G, method=CONTAINMENT):
    atoms = [eb.Prop(atom) for atom in G.atoms()]
    sorted_atoms = tuple(sorted(atoms))

    ctl = gringo.Control()
    #ctl.conf.configuration = 'handy'
    ctl.conf.solver.heuristic = 'domain'
    ctl.conf.solve.enum_mode = 'domRec'
    ctl.conf.solve.models = 0
    #ctl.conf.solve.parallel_mode = 2

    ctl.load(EQ_SETS_FILE)
    ctl.add('base', [], eb.convert_to_asp(G))
    ctl.ground([('base', [])])

    node_models = defaultdict(set)
    node_tv_dict = defaultdict(int)

    it = ctl.solve_iter()
    for m in it:
        node_tv_dict.clear()
        terms = m.atoms(gringo.Model.SHOWN)
        for term in terms:
            if term.name() == 'tv':
                node, atom_index, truth_value = term.args()
                node_tv_dict[node] |= truth_value << atom_index

        for node in node_tv_dict:
            node_models[node].add(node_tv_dict[node])
    print(node_models)

    R = copy.deepcopy(G)
    for node in node_models:
        t = tuple(node_models[node])
        formula = formula_from_models(t, sorted_atoms)
        simple = simplify(formula)
        R.set_formulas(node, [simple])
    return R

def memo(func):
    cache = {}
    @wraps(func)
    def wrap(*args):
        if args not in cache:
            cache[args] = func(*args)
        return cache[args]
    return wrap

@memo
def simplify(formula):
    return eb.simplify(formula)

@memo
def formula_from_models(models, atoms):
    disjunction = eb.Prop(False)
    for model in models:
        disjunction |= formula_from_model(model, atoms)
    return disjunction

@memo
def formula_from_model(model, atoms):
    """Creates a propositional formula from a (single) model.

       The input model must be an integer whose bits represent
       the truth values of atoms in a given sorted order.

       For example, given an alphabet A = {p, q, r}, and a
       model 6 = 110 (base 2), the formula returned by this
       function is p & q & ~r.
    """
    conjunction = eb.Prop(True)
    for (index, atom) in enumerate(atoms):
        truth_value = model & (1 << index)
        if truth_value:
            conjunction &= atom
        else:
            conjunction &= ~atom
    return conjunction

def revise(K, R):
    G = equibel.path_graph(2)

    if isinstance(K, str):
        G.add_formula(0, equibel.parse_infix_formula(K))
    elif isinstance(K, list):
        for belief in K:
            G.add_formula(0, equibel.parse_infix_formula(belief))

    if isinstance(R, str):
        G.add_formula(1, equibel.parse_infix_formula(R))
    elif isinstance(R, list):
        for belief in R:
            G.add_formula(1, equibel.parse_infix_formula(belief))

    S = completion(G)
    return S.formulas(1)

def proj_merge(belief_bases, entailment_constraint=None):
    G = equibel.star_graph(len(belief_bases))

    if entailment_constraint:
        G.add_formula(0, entailment_constraint)

    for (i, belief_base) in enumerate(belief_bases, 1):
        G.add_formula(i, belief_base)

    S = completion(G)
    return S.formulas(0)

def con_merge(belief_bases, entailment_based_constraints=None, consistency_based_constraints=None):
    pass

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("usage: python solver.py NUM_NODES")
        sys.exit(1)

    num_nodes = int(sys.argv[1])

    solver = EqSolver()
    G = equibel.path_graph(num_nodes)
    G.add_formula(0, "p & q & r & s")
    G.add_formula(num_nodes-1, "~p & ~q & ~r & ~s")
    R = completion(G)
    for node_id in R.nodes():
        print("Node {0}, formulas = {1}".format(node_id, R.formulas(node_id)))
<commit_msg>Added absolute imports<commit_after>from __future__ import absolute_import

import sys
from collections import defaultdict
from functools import wraps
import copy
import platform
import pkg_resources

import gringo

import equibel as eb

if platform.system() == 'Linux':
    if platform.architecture()[0] == '64bit':
        if platform.dist()[0] in ['centos', 'CentOS']:
            import equibel.includes.linux.bit64.centos.gringo as gringo
            from equibel.includes.linux.bit64.centos.gringo import Control, Model, Fun
        else:
            import equibel.includes.linux.bit64.gringo as gringo
            from equibel.includes.linux.bit64.gringo import Control, Model, Fun
    elif platform.architecture()[0] == '32bit':
        import equibel.includes.linux.bit32.gringo as gringo
        from equibel.includes.linux.bit32.gringo import Control, Model, Fun
elif platform.system() == 'Darwin':
    import equibel.includes.mac.gringo as gringo
    from equibel.includes.mac.gringo import Control, Model, Fun

import equibel.FormulaExtractor as FormulaExtractor
import equibel.formatters.ASP_Formatter as ASP_Formatter
#from equibel.simbool.simplify import simplify

EQ_SETS_FILE = pkg_resources.resource_filename('equibel', 'asp/eq_sets.lp')

CONTAINMENT = 0
CARDINALITY = 1

def completion(G, method=CONTAINMENT):
    atoms = [eb.Prop(atom) for atom in G.atoms()]
    sorted_atoms = tuple(sorted(atoms))

    ctl = gringo.Control()
    #ctl.conf.configuration = 'handy'
    ctl.conf.solver.heuristic = 'domain'
    ctl.conf.solve.enum_mode = 'domRec'
    ctl.conf.solve.models = 0
    #ctl.conf.solve.parallel_mode = 2

    ctl.load(EQ_SETS_FILE)
    ctl.add('base', [], eb.convert_to_asp(G))
    ctl.ground([('base', [])])

    node_models = defaultdict(set)
    node_tv_dict = defaultdict(int)

    it = ctl.solve_iter()
    for m in it:
        node_tv_dict.clear()
        terms = m.atoms(gringo.Model.SHOWN)
        for term in terms:
            if term.name() == 'tv':
                node, atom_index, truth_value = term.args()
                node_tv_dict[node] |= truth_value << atom_index

        for node in node_tv_dict:
            node_models[node].add(node_tv_dict[node])
    print(node_models)

    R = copy.deepcopy(G)
    for node in node_models:
        t = tuple(node_models[node])
        formula = formula_from_models(t, sorted_atoms)
        simple = simplify(formula)
        R.set_formulas(node, [simple])
    return R

def memo(func):
    cache = {}
    @wraps(func)
    def wrap(*args):
        if args not in cache:
            cache[args] = func(*args)
        return cache[args]
    return wrap

@memo
def simplify(formula):
    return eb.simplify(formula)

@memo
def formula_from_models(models, atoms):
    disjunction = eb.Prop(False)
    for model in models:
        disjunction |= formula_from_model(model, atoms)
    return disjunction

@memo
def formula_from_model(model, atoms):
    """Creates a propositional formula from a (single) model.

       The input model must be an integer whose bits represent
       the truth values of atoms in a given sorted order.

       For example, given an alphabet A = {p, q, r}, and a
       model 6 = 110 (base 2), the formula returned by this
       function is p & q & ~r.
    """
    conjunction = eb.Prop(True)
    for (index, atom) in enumerate(atoms):
        truth_value = model & (1 << index)
        if truth_value:
            conjunction &= atom
        else:
            conjunction &= ~atom
    return conjunction

def revise(K, R):
    G = equibel.path_graph(2)

    if isinstance(K, str):
        G.add_formula(0, equibel.parse_infix_formula(K))
    elif isinstance(K, list):
        for belief in K:
            G.add_formula(0, equibel.parse_infix_formula(belief))

    if isinstance(R, str):
        G.add_formula(1, equibel.parse_infix_formula(R))
    elif isinstance(R, list):
        for belief in R:
            G.add_formula(1, equibel.parse_infix_formula(belief))

    S = completion(G)
    return S.formulas(1)

def proj_merge(belief_bases, entailment_constraint=None):
    G = equibel.star_graph(len(belief_bases))

    if entailment_constraint:
        G.add_formula(0, entailment_constraint)

    for (i, belief_base) in enumerate(belief_bases, 1):
        G.add_formula(i, belief_base)

    S = completion(G)
    return S.formulas(0)

def con_merge(belief_bases, entailment_based_constraints=None, consistency_based_constraints=None):
    pass

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("usage: python solver.py NUM_NODES")
        sys.exit(1)

    num_nodes = int(sys.argv[1])

    solver = EqSolver()
    G = equibel.path_graph(num_nodes)
    G.add_formula(0, "p & q & r & s")
    G.add_formula(num_nodes-1, "~p & ~q & ~r & ~s")
    R = completion(G)
    for node_id in R.nodes():
        print("Node {0}, formulas = {1}".format(node_id, R.formulas(node_id)))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#    Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Base',
    'version': '1.2',
    'category': 'Generic Modules/Base',
    'description': """The kernel of OpenERP, needed for all installation.""",
    'author': 'OpenERP s.a.',
    'maintainer': 'OpenERP s.a.',
    'website': 'http://www.openerp.com',
    'depends': [],
    'init_xml': [
        'base_data.xml',
        'security/base_security.xml',
        'base_menu.xml',
        'res/res_security.xml',
        'res/res_config.xml',
        'data/res.country.state.csv'
    ],
    'update_xml': [
        'base_update.xml',
        'ir/wizard/wizard_menu_view.xml',
        'ir/ir.xml',
        'ir/workflow/workflow_view.xml',
        'module/module_wizard.xml',
        'module/module_view.xml',
        'module/module_web_view.xml',
        'module/module_data.xml',
        'module/module_report.xml',
        'res/res_request_view.xml',
        'res/res_lang_view.xml',
        'res/res_log_view.xml',
        'res/res_company_view.xml',
        'res/partner/partner_report.xml',
        'res/partner/partner_view.xml',
        'res/partner/partner_wizard.xml',
        'res/bank_view.xml',
        'res/country_view.xml',
        'res/res_currency_view.xml',
        'res/partner/crm_view.xml',
        'res/partner/wizard/partner_sms_send_view.xml',
        'res/partner/partner_data.xml',
        'res/ir_property_view.xml',
        'security/base_security.xml',
        'maintenance/maintenance_view.xml',
        'security/ir.model.access.csv'

    ],
    'demo_xml': [
        'base_demo.xml',
        'res/partner/partner_demo.xml',
        'res/partner/crm_demo.xml',
    ],
    'test': [
        'test/base_test.xml',
        #'test/base_test.yml'
        'test/test_context.xml',
        'bug_lp541545.xml',
    ],
    'installable': True,
    'active': True,
    'certificate': '0076807797149',
}
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>Base: remove ref to non-existent test file.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#    Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Base',
    'version': '1.2',
    'category': 'Generic Modules/Base',
    'description': """The kernel of OpenERP, needed for all installation.""",
    'author': 'OpenERP s.a.',
    'maintainer': 'OpenERP s.a.',
    'website': 'http://www.openerp.com',
    'depends': [],
    'init_xml': [
        'base_data.xml',
        'security/base_security.xml',
        'base_menu.xml',
        'res/res_security.xml',
        'res/res_config.xml',
        'data/res.country.state.csv'
    ],
    'update_xml': [
        'base_update.xml',
        'ir/wizard/wizard_menu_view.xml',
        'ir/ir.xml',
        'ir/workflow/workflow_view.xml',
        'module/module_wizard.xml',
        'module/module_view.xml',
        'module/module_web_view.xml',
        'module/module_data.xml',
        'module/module_report.xml',
        'res/res_request_view.xml',
        'res/res_lang_view.xml',
        'res/res_log_view.xml',
        'res/res_company_view.xml',
        'res/partner/partner_report.xml',
        'res/partner/partner_view.xml',
        'res/partner/partner_wizard.xml',
        'res/bank_view.xml',
        'res/country_view.xml',
        'res/res_currency_view.xml',
        'res/partner/crm_view.xml',
        'res/partner/wizard/partner_sms_send_view.xml',
        'res/partner/partner_data.xml',
        'res/ir_property_view.xml',
        'security/base_security.xml',
        'maintenance/maintenance_view.xml',
        'security/ir.model.access.csv'

    ],
    'demo_xml': [
        'base_demo.xml',
        'res/partner/partner_demo.xml',
        'res/partner/crm_demo.xml',
    ],
    'test': [
        'test/base_test.xml',
        #'test/base_test.yml'
        'test/test_context.xml',
    ],
    'installable': True,
    'active': True,
    'certificate': '0076807797149',
}
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    These preprocessing utils would greatly benefit
    from a fast Cython rewrite.
'''
from __future__ import absolute_import

import string
import sys
import numpy as np
from six.moves import range
from six.moves import zip

if sys.version_info < (3,):
    maketrans = string.maketrans
else:
    maketrans = str.maketrans

def base_filter():
    f = string.punctuation
    f = f.replace("'", '')
    f += '\t\n'
    return f

def text_to_word_sequence(text, filters=base_filter(), lower=True, split=" "):
    '''prune: sequence of characters to filter out
    '''
    if lower:
        text = text.lower()
    text = text.translate(maketrans(filters, split*len(filters)))
    seq = text.split(split)
    return [_f for _f in seq if _f]

def one_hot(text, n, filters=base_filter(), lower=True, split=" "):
    seq = text_to_word_sequence(text, filters=filters, lower=lower, split=split)
    return [(abs(hash(w)) % (n - 1) + 1) for w in seq]

class Tokenizer(object):
    def __init__(self, nb_words=None, filters=base_filter(),
                 lower=True, split=' ', char_level=False):
        '''The class allows to vectorize a text corpus, by turning each
        text into either a sequence of integers (each integer being the index
        of a token in a dictionary) or into a vector where the coefficient
        for each token could be binary, based on word count, based on tf-idf...

        # Arguments
            nb_words: the maximum number of words to keep, based
                on word frequency. Only the most common `nb_words` words will
                be kept.
            filters: a string where each element is a character that will be
                filtered from the texts. The default is all punctuation, plus
                tabs and line breaks, minus the `'` character.
            lower: boolean. Whether to convert the texts to lowercase.
            split: character or string to use for token splitting.

        By default, all punctuation is removed, turning the texts into
        space-separated sequences of words
        (words maybe include the `'` character). These sequences are then
        split into lists of tokens. They will then be indexed or vectorized.

        `0` is a reserved index that won't be assigned to any word.
        '''
        self.word_counts = {}
        self.word_docs = {}
        self.filters = filters
        self.split = split
        self.lower = lower
        self.nb_words = nb_words
        self.document_count = 0
        self.char_level = char_level

    def fit_on_texts(self, texts):
        '''
            required before using texts_to_sequences or texts_to_matrix

        # Arguments
            texts: can be a list of strings,
                or a generator of strings (for memory-efficiency)
        '''
        self.document_count = 0
        for text in texts:
            self.document_count += 1
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            for w in seq:
                if w in self.word_counts:
                    self.word_counts[w] += 1
                else:
                    self.word_counts[w] = 1
            for w in set(seq):
                if w in self.word_docs:
                    self.word_docs[w] += 1
                else:
                    self.word_docs[w] = 1

        wcounts = list(self.word_counts.items())
        wcounts.sort(key=lambda x: x[1], reverse=True)
        sorted_voc = [wc[0] for wc in wcounts]
        self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))

        self.index_docs = {}
        for w, c in list(self.word_docs.items()):
            self.index_docs[self.word_index[w]] = c

    def fit_on_sequences(self, sequences):
        '''
            required before using sequences_to_matrix
            (if fit_on_texts was never called)
        '''
        self.document_count = len(sequences)
        self.index_docs = {}
        for seq in sequences:
            seq = set(seq)
            for i in seq:
                if i not in self.index_docs:
                    self.index_docs[i] = 1
                else:
                    self.index_docs[i] += 1

    def texts_to_sequences(self, texts):
        '''
            Transform each text in texts in a sequence of integers.
            Only top "nb_words" most frequent words will be taken into account.
            Only words known by the tokenizer will be taken into account.

            Returns a list of sequences.
        '''
        res = []
        for vect in self.texts_to_sequences_generator(texts):
            res.append(vect)
        return res

    def texts_to_sequences_generator(self, texts):
        '''
            Transform each text in texts in a sequence of integers.
            Only top "nb_words" most frequent words will be taken into account.
            Only words known by the tokenizer will be taken into account.

            Yields individual sequences.
        '''
        nb_words = self.nb_words
        for text in texts:
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            vect = []
            for w in seq:
                i = self.word_index.get(w)
                if i is not None:
                    if nb_words and i >= nb_words:
                        pass
                    else:
                        vect.append(i)
            yield vect

    def texts_to_matrix(self, texts, mode="binary"):
        '''
            modes: binary, count, tfidf, freq
        '''
        sequences = self.texts_to_sequences(texts)
        return self.sequences_to_matrix(sequences, mode=mode)

    def sequences_to_matrix(self, sequences, mode="binary"):
        '''
            modes: binary, count, tfidf, freq
        '''
        if not self.nb_words:
            if self.word_index:
                nb_words = len(self.word_index) + 1
            else:
                raise Exception("Specify a dimension (nb_words argument), or fit on some text data first.")
        else:
            nb_words = self.nb_words

        if mode == "tfidf" and not self.document_count:
            raise Exception("Fit the Tokenizer on some data before using tfidf mode.")

        X = np.zeros((len(sequences), nb_words))
        for i, seq in enumerate(sequences):
            if not seq:
                pass
            counts = {}
            for j in seq:
                if j >= nb_words:
                    pass
                if j not in counts:
                    counts[j] = 1.
                else:
                    counts[j] += 1
            for j, c in list(counts.items()):
                if mode == "count":
                    X[i][j] = c
                elif mode == "freq":
                    X[i][j] = c / len(seq)
                elif mode == "binary":
                    X[i][j] = 1
                elif mode == "tfidf":
                    tf = np.log(c / len(seq))
                    df = (1 + np.log(1 + self.index_docs.get(j, 0) / (1 + self.document_count)))
                    X[i][j] = tf / df
                else:
                    raise Exception("Unknown vectorization mode: " + str(mode))
        return X
<commit_msg>add doc<commit_after># -*- coding: utf-8 -*-
'''
    These preprocessing utils would greatly benefit
    from a fast Cython rewrite.
'''
from __future__ import absolute_import

import string
import sys
import numpy as np
from six.moves import range
from six.moves import zip

if sys.version_info < (3,):
    maketrans = string.maketrans
else:
    maketrans = str.maketrans

def base_filter():
    f = string.punctuation
    f = f.replace("'", '')
    f += '\t\n'
    return f

def text_to_word_sequence(text, filters=base_filter(), lower=True, split=" "):
    '''prune: sequence of characters to filter out
    '''
    if lower:
        text = text.lower()
    text = text.translate(maketrans(filters, split*len(filters)))
    seq = text.split(split)
    return [_f for _f in seq if _f]

def one_hot(text, n, filters=base_filter(), lower=True, split=" "):
    seq = text_to_word_sequence(text, filters=filters, lower=lower, split=split)
    return [(abs(hash(w)) % (n - 1) + 1) for w in seq]

class Tokenizer(object):
    def __init__(self, nb_words=None, filters=base_filter(),
                 lower=True, split=' ', char_level=False):
        '''The class allows to vectorize a text corpus, by turning each
        text into either a sequence of integers (each integer being the index
        of a token in a dictionary) or into a vector where the coefficient
        for each token could be binary, based on word count, based on tf-idf...

        # Arguments
            nb_words: the maximum number of words to keep, based
                on word frequency. Only the most common `nb_words` words will
                be kept.
            filters: a string where each element is a character that will be
                filtered from the texts. The default is all punctuation, plus
                tabs and line breaks, minus the `'` character.
            lower: boolean. Whether to convert the texts to lowercase.
            split: character or string to use for token splitting.
            char_level: if True, every character will be treated as a word.

        By default, all punctuation is removed, turning the texts into
        space-separated sequences of words
        (words maybe include the `'` character). These sequences are then
        split into lists of tokens. They will then be indexed or vectorized.

        `0` is a reserved index that won't be assigned to any word.
        '''
        self.word_counts = {}
        self.word_docs = {}
        self.filters = filters
        self.split = split
        self.lower = lower
        self.nb_words = nb_words
        self.document_count = 0
        self.char_level = char_level

    def fit_on_texts(self, texts):
        '''
            required before using texts_to_sequences or texts_to_matrix

        # Arguments
            texts: can be a list of strings,
                or a generator of strings (for memory-efficiency)
        '''
        self.document_count = 0
        for text in texts:
            self.document_count += 1
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            for w in seq:
                if w in self.word_counts:
                    self.word_counts[w] += 1
                else:
                    self.word_counts[w] = 1
            for w in set(seq):
                if w in self.word_docs:
                    self.word_docs[w] += 1
                else:
                    self.word_docs[w] = 1

        wcounts = list(self.word_counts.items())
        wcounts.sort(key=lambda x: x[1], reverse=True)
        sorted_voc = [wc[0] for wc in wcounts]
        self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))

        self.index_docs = {}
        for w, c in list(self.word_docs.items()):
            self.index_docs[self.word_index[w]] = c

    def fit_on_sequences(self, sequences):
        '''
            required before using sequences_to_matrix
            (if fit_on_texts was never called)
        '''
        self.document_count = len(sequences)
        self.index_docs = {}
        for seq in sequences:
            seq = set(seq)
            for i in seq:
                if i not in self.index_docs:
                    self.index_docs[i] = 1
                else:
                    self.index_docs[i] += 1

    def texts_to_sequences(self, texts):
        '''
            Transform each text in texts in a sequence of integers.
            Only top "nb_words" most frequent words will be taken into account.
            Only words known by the tokenizer will be taken into account.

            Returns a list of sequences.
        '''
        res = []
        for vect in self.texts_to_sequences_generator(texts):
            res.append(vect)
        return res

    def texts_to_sequences_generator(self, texts):
        '''
            Transform each text in texts in a sequence of integers.
            Only top "nb_words" most frequent words will be taken into account.
            Only words known by the tokenizer will be taken into account.

            Yields individual sequences.
        '''
        nb_words = self.nb_words
        for text in texts:
            seq = text if self.char_level else text_to_word_sequence(text, self.filters, self.lower, self.split)
            vect = []
            for w in seq:
                i = self.word_index.get(w)
                if i is not None:
                    if nb_words and i >= nb_words:
                        pass
                    else:
                        vect.append(i)
            yield vect

    def texts_to_matrix(self, texts, mode="binary"):
        '''
            modes: binary, count, tfidf, freq
        '''
        sequences = self.texts_to_sequences(texts)
        return self.sequences_to_matrix(sequences, mode=mode)

    def sequences_to_matrix(self, sequences, mode="binary"):
        '''
            modes: binary, count, tfidf, freq
        '''
        if not self.nb_words:
            if self.word_index:
                nb_words = len(self.word_index) + 1
            else:
                raise Exception("Specify a dimension (nb_words argument), or fit on some text data first.")
        else:
            nb_words = self.nb_words

        if mode == "tfidf" and not self.document_count:
            raise Exception("Fit the Tokenizer on some data before using tfidf mode.")

        X = np.zeros((len(sequences), nb_words))
        for i, seq in enumerate(sequences):
            if not seq:
                pass
            counts = {}
            for j in seq:
                if j >= nb_words:
                    pass
                if j not in counts:
                    counts[j] = 1.
                else:
                    counts[j] += 1
            for j, c in list(counts.items()):
                if mode == "count":
                    X[i][j] = c
                elif mode == "freq":
                    X[i][j] = c / len(seq)
                elif mode == "binary":
                    X[i][j] = 1
                elif mode == "tfidf":
                    tf = np.log(c / len(seq))
                    df = (1 + np.log(1 + self.index_docs.get(j, 0) / (1 + self.document_count)))
                    X[i][j] = tf / df
                else:
                    raise Exception("Unknown vectorization mode: " + str(mode))
        return X
<|endoftext|>