<commit_before>from sqlalchemy.orm.exc import NoResultFound,MultipleResultsFound
from dataactcore.utils.responseException import ResponseException
from dataactcore.models.errorModels import FileStatus, ErrorData
from dataactcore.models.errorInterface import ErrorInterface
from dataactvalidator.validation_handlers.validationError import ValidationError

class ValidatorErrorInterface(ErrorInterface):
    """ Manages communication with the error database """

    def __init__(self):
        """ Create empty row error dict """
        self.rowErrors = {}
        super(ValidatorErrorInterface, self).__init__()

    def createFileStatus(self,jobId, filename):
        """ Create a new file status object for specified job and filename """
        try:
            int(jobId)
        except:
            raise ValueError("".join(["Bad jobId: ",str(jobId)]))

        fileStatus = FileStatus(job_id = jobId, filename = filename, row_errors_present = False)
        self.session.add(fileStatus)
        self.session.commit()
        return fileStatus

    def createFileStatusIfNeeded(self, jobId, filename):
        """ Return the existing FileStatus if it exists, or create a new one """
        try:
            fileStatus = self.getFileStatusByJobId(jobId)
            if fileStatus.filename != filename:
                # There is a FileStatus for this job, but the filename does not match
                raise ValueError("".join(["FileStatus exists for this Job ID, but old filename ",fileStatus.filename," does not match new filename ",filename]))
        except ResponseException as e:
            if isinstance(e.wrappedException, NoResultFound):
                # No File Status object for this job ID, just create one
                fileStatus = self.createFileStatus(jobId, filename)
            else:
                # Other error types should be handled at a higher level, so re-raise
                raise
        return fileStatus

    def writeFileError(self, jobId, filename, errorType, extraInfo = None):
        """ Write a file-level error to the file status table

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3
            errorType: type of error, value will be mapped to ValidationError class

        Returns:
            True if successful
        """
        try:
            int(jobId)
        except:
            raise ValueError("".join(["Bad jobId: ",str(jobId)]))

        # Get File Status for this job ID or create it if it doesn't exist
        fileStatus = self.createFileStatusIfNeeded(jobId,filename)

        # Mark error type and add header info if present
        fileStatus.status_id = self.getStatusId(ValidationError.getErrorTypeString(errorType))
        if extraInfo is not None:
            if "missing_headers" in extraInfo:
                fileStatus.headers_missing = extraInfo["missing_headers"]
            if "duplicated_headers" in extraInfo:
                fileStatus.headers_duplicated = extraInfo["duplicated_headers"]

        self.session.add(fileStatus)
        self.session.commit()
        return True

    def markFileComplete(self, jobId, filename):
        """ Marks file status as complete

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3

        Returns:
            True if successful
        """

        fileComplete = self.createFileStatusIfNeeded(jobId,filename)
        fileComplete.status_id = self.getStatusId("complete")
        self.session.commit()
        return True

    def recordRowError(self, jobId, filename, fieldName, errorType, row):
        """ Add this error to running sum of error types

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3
            fieldName: name of field where error occurred
            errorType: type of error, value will be mapped to ValidationError class, for rule failures this will hold entire message

        Returns:
            True if successful
        """
        key = "".join([str(jobId),fieldName,str(errorType)])
        if(key in self.rowErrors):
            self.rowErrors[key]["numErrors"] += 1
        else:
            errorDict = {"filename":filename, "fieldName":fieldName, "jobId":jobId,"errorType":errorType,"numErrors":1, "firstRow":row}
            self.rowErrors[key] = errorDict

    def writeAllRowErrors(self, jobId):
        """ Writes all recorded errors to database

        Args:
            jobId: ID to write errors for

        Returns:
            True if successful
        """
        for key in self.rowErrors.keys():
            errorDict = self.rowErrors[key]
            # Set info for this error
            thisJob = errorDict["jobId"]
            if(int(jobId) != int(thisJob)):
                # This row is for a different job, skip it
                continue
            fieldName = errorDict["fieldName"]
            try:
                # If last part of key is an int, it's one of our prestored messages
                errorType = int(errorDict["errorType"])
            except ValueError:
                # For rule failures, it will hold the error message
                errorMsg = errorDict["errorType"]
                ruleFailedId = self.getTypeId("rule_failed")
                errorRow = ErrorData(job_id = thisJob, filename = errorDict["filename"], field_name = fieldName, error_type_id = ruleFailedId, rule_failed = errorMsg, occurrences = errorDict["numErrors"], first_row = errorDict["firstRow"])
            else:
                # This happens if cast to int was successful
                errorString = ValidationError.getErrorTypeString(errorType)
                errorId = self.getTypeId(errorString)
                # Create error data
                errorRow = ErrorData(job_id = thisJob, filename = errorDict["filename"], field_name = fieldName, error_type_id = errorId, occurrences = errorDict["numErrors"], first_row = errorDict["firstRow"], rule_failed = ValidationError.getErrorMessage(errorType))

            self.session.add(errorRow)

        # Commit the session to write all rows
        self.session.commit()
        # Clear the dictionary
        self.rowErrors = {}

    def writeMissingHeaders(self, jobId, missingHeaders):
        """ Write list of missing headers into headers_missing field

        Args:
            jobId: Job to write error for
            missingHeaders: List of missing headers

        """
        fileStatus = self.getFileStatusByJobId(jobId)
        # Create single string out of missing header list
        fileStatus.headers_missing = ",".join(missingHeaders)
        self.session.commit()

    def writeDuplicatedHeaders(self, jobId, duplicatedHeaders):
        """ Write list of duplicated headers into headers_missing field

        Args:
            jobId: Job to write error for
            duplicatedHeaders: List of duplicated headers

        """
        fileStatus = self.getFileStatusByJobId(jobId)
        # Create single string out of duplicated header list
        fileStatus.headers_duplicated = ",".join(duplicatedHeaders)
        self.session.commit()

    def setRowErrorsPresent(self, jobId, errorsPresent):
        """ Set errors present for the specified job ID to true or false.  Note this refers only to row-level errors, not file-level errors. """
        fileStatus = self.getFileStatusByJobId(jobId)
        # If errorsPresent is not a bool, this function will raise a TypeError
        fileStatus.row_errors_present = bool(errorsPresent)
        self.session.commit()

    def getRowErrorsPresent(self, jobId):
        """ Returns True or False depending on if errors were found in the specified job """
        return self.getFileStatusByJobId(jobId).row_errors_present<commit_msg>Adding a file status from start of validation<commit_after>from sqlalchemy.orm.exc import NoResultFound,MultipleResultsFound
from dataactcore.utils.responseException import ResponseException
from dataactcore.models.errorModels import FileStatus, ErrorData
from dataactcore.models.errorInterface import ErrorInterface
from dataactvalidator.validation_handlers.validationError import ValidationError

class ValidatorErrorInterface(ErrorInterface):
    """ Manages communication with the error database """

    def __init__(self):
        """ Create empty row error dict """
        self.rowErrors = {}
        super(ValidatorErrorInterface, self).__init__()

    def createFileStatus(self,jobId, filename):
        """ Create a new file status object for specified job and filename """
        try:
            int(jobId)
        except:
            raise ValueError("".join(["Bad jobId: ",str(jobId)]))

        fileStatus = FileStatus(job_id = jobId, filename = filename, row_errors_present = False, status_id = self.getStatusId("incomplete"))
        self.session.add(fileStatus)
        self.session.commit()
        return fileStatus

    def createFileStatusIfNeeded(self, jobId, filename):
        """ Return the existing FileStatus if it exists, or create a new one """
        try:
            fileStatus = self.getFileStatusByJobId(jobId)
            if fileStatus.filename != filename:
                # There is a FileStatus for this job, but the filename does not match
                raise ValueError("".join(["FileStatus exists for this Job ID, but old filename ",fileStatus.filename," does not match new filename ",filename]))
        except ResponseException as e:
            if isinstance(e.wrappedException, NoResultFound):
                # No File Status object for this job ID, just create one
                fileStatus = self.createFileStatus(jobId, filename)
            else:
                # Other error types should be handled at a higher level, so re-raise
                raise
        return fileStatus

    def writeFileError(self, jobId, filename, errorType, extraInfo = None):
        """ Write a file-level error to the file status table

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3
            errorType: type of error, value will be mapped to ValidationError class

        Returns:
            True if successful
        """
        try:
            int(jobId)
        except:
            raise ValueError("".join(["Bad jobId: ",str(jobId)]))

        # Get File Status for this job ID or create it if it doesn't exist
        fileStatus = self.createFileStatusIfNeeded(jobId,filename)

        # Mark error type and add header info if present
        fileStatus.status_id = self.getStatusId(ValidationError.getErrorTypeString(errorType))
        if extraInfo is not None:
            if "missing_headers" in extraInfo:
                fileStatus.headers_missing = extraInfo["missing_headers"]
            if "duplicated_headers" in extraInfo:
                fileStatus.headers_duplicated = extraInfo["duplicated_headers"]

        self.session.add(fileStatus)
        self.session.commit()
        return True

    def markFileComplete(self, jobId, filename):
        """ Marks file status as complete

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3

        Returns:
            True if successful
        """

        fileComplete = self.createFileStatusIfNeeded(jobId,filename)
        fileComplete.status_id = self.getStatusId("complete")
        self.session.commit()
        return True

    def recordRowError(self, jobId, filename, fieldName, errorType, row):
        """ Add this error to running sum of error types

        Args:
            jobId: ID of job in job tracker
            filename: name of error report in S3
            fieldName: name of field where error occurred
            errorType: type of error, value will be mapped to ValidationError class, for rule failures this will hold entire message

        Returns:
            True if successful
        """
        key = "".join([str(jobId),fieldName,str(errorType)])
        if(key in self.rowErrors):
            self.rowErrors[key]["numErrors"] += 1
        else:
            errorDict = {"filename":filename, "fieldName":fieldName, "jobId":jobId,"errorType":errorType,"numErrors":1, "firstRow":row}
            self.rowErrors[key] = errorDict

    def writeAllRowErrors(self, jobId):
        """ Writes all recorded errors to database

        Args:
            jobId: ID to write errors for

        Returns:
            True if successful
        """
        for key in self.rowErrors.keys():
            errorDict = self.rowErrors[key]
            # Set info for this error
            thisJob = errorDict["jobId"]
            if(int(jobId) != int(thisJob)):
                # This row is for a different job, skip it
                continue
            fieldName = errorDict["fieldName"]
            try:
                # If last part of key is an int, it's one of our prestored messages
                errorType = int(errorDict["errorType"])
            except ValueError:
                # For rule failures, it will hold the error message
                errorMsg = errorDict["errorType"]
                ruleFailedId = self.getTypeId("rule_failed")
                errorRow = ErrorData(job_id = thisJob, filename = errorDict["filename"], field_name = fieldName, error_type_id = ruleFailedId, rule_failed = errorMsg, occurrences = errorDict["numErrors"], first_row = errorDict["firstRow"])
            else:
                # This happens if cast to int was successful
                errorString = ValidationError.getErrorTypeString(errorType)
                errorId = self.getTypeId(errorString)
                # Create error data
                errorRow = ErrorData(job_id = thisJob, filename = errorDict["filename"], field_name = fieldName, error_type_id = errorId, occurrences = errorDict["numErrors"], first_row = errorDict["firstRow"], rule_failed = ValidationError.getErrorMessage(errorType))

            self.session.add(errorRow)

        # Commit the session to write all rows
        self.session.commit()
        # Clear the dictionary
        self.rowErrors = {}

    def writeMissingHeaders(self, jobId, missingHeaders):
        """ Write list of missing headers into headers_missing field

        Args:
            jobId: Job to write error for
            missingHeaders: List of missing headers

        """
        fileStatus = self.getFileStatusByJobId(jobId)
        # Create single string out of missing header list
        fileStatus.headers_missing = ",".join(missingHeaders)
        self.session.commit()

    def writeDuplicatedHeaders(self, jobId, duplicatedHeaders):
        """ Write list of duplicated headers into headers_missing field

        Args:
            jobId: Job to write error for
            duplicatedHeaders: List of duplicated headers

        """
        fileStatus = self.getFileStatusByJobId(jobId)
        # Create single string out of duplicated header list
        fileStatus.headers_duplicated = ",".join(duplicatedHeaders)
        self.session.commit()

    def setRowErrorsPresent(self, jobId, errorsPresent):
        """ Set errors present for the specified job ID to true or false.  Note this refers only to row-level errors, not file-level errors. """
        fileStatus = self.getFileStatusByJobId(jobId)
        # If errorsPresent is not a bool, this function will raise a TypeError
        fileStatus.row_errors_present = bool(errorsPresent)
        self.session.commit()

    def getRowErrorsPresent(self, jobId):
        """ Returns True or False depending on if errors were found in the specified job """
        return self.getFileStatusByJobId(jobId).row_errors_present<|endoftext|>


<commit_before>"""
This example show how in-document search can be used with a CrossEncoder.

The document is split into passage. Here, we use three consecutive sentences as a passage. You can use shorter passage, for example, individual sentences,
or longer passages, like full paragraphs.

The CrossEncoder takes the search query and scores every passage how relevant the passage is for the given score. The five passages with the highest score are  then returned.

As CrossEncoder, we use sentence-transformers/ce-ms-marco-TinyBERT-L-2, a BERT model with only 2 layers trained on the MS MARCO dataset. This is an extremely quick model able to score up to 9000 passages per second (on a V100 GPU). You can also use a larger model, which gives better results but is also slower.

Note: As we score the [query, passage]-pair for every new query, this search method
becomes at some point in-efficient if the document gets too large.

Usage: python in_document_search_crossencoder.py
"""

from sentence_transformers import CrossEncoder
from nltk import sent_tokenize
import time

#As document, we take the first two section from the Wikipedia article about Europe
document = """Europe is a continent located entirely in the Northern Hemisphere and mostly in the Eastern Hemisphere. It comprises the westernmost part of Eurasia and is bordered by the Arctic Ocean to the north, the Atlantic Ocean to the west, the Mediterranean Sea to the south, and Asia to the east. Europe is commonly considered to be separated from Asia by the watershed of the Ural Mountains, the Ural River, the Caspian Sea, the Greater Caucasus, the Black Sea, and the waterways of the Turkish Straits.[11] Although some of this border is over land, Europe is generally accorded the status of a full continent because of its great physical size and the weight of history and tradition.

Europe covers about 10,180,000 square kilometres (3,930,000 sq mi), or 2% of the Earth's surface (6.8% of land area), making it the second smallest continent. Politically, Europe is divided into about fifty sovereign states, of which Russia is the largest and most populous, spanning 39% of the continent and comprising 15% of its population. Europe had a total population of about 741 million (about 11% of the world population) as of 2018.[2][3] The European climate is largely affected by warm Atlantic currents that temper winters and summers on much of the continent, even at latitudes along which the climate in Asia and North America is severe. Further from the sea, seasonal differences are more noticeable than close to the coast.

European culture is the root of Western civilization, which traces its lineage back to ancient Greece and ancient Rome.[12][13] The fall of the Western Roman Empire in 476 AD and the subsequent Migration Period marked the end of Europe's ancient history and the beginning of the Middle Ages. Renaissance humanism, exploration, art and science led to the modern era. Since the Age of Discovery, started by Portugal and Spain, Europe played a predominant role in global affairs. Between the 16th and 20th centuries, European powers colonized at various times the Americas, almost all of Africa and Oceania, and the majority of Asia.

The Age of Enlightenment, the subsequent French Revolution and the Napoleonic Wars shaped the continent culturally, politically and economically from the end of the 17th century until the first half of the 19th century. The Industrial Revolution, which began in Great Britain at the end of the 18th century, gave rise to radical economic, cultural and social change in Western Europe and eventually the wider world. Both world wars took place for the most part in Europe, contributing to a decline in Western European dominance in world affairs by the mid-20th century as the Soviet Union and the United States took prominence.[14] During the Cold War, Europe was divided along the Iron Curtain between NATO in the West and the Warsaw Pact in the East, until the revolutions of 1989 and fall of the Berlin Wall.

In 1949, the Council of Europe was founded with the idea of unifying Europe to achieve common goals. Further European integration by some states led to the formation of the European Union (EU), a separate political entity that lies between a confederation and a federation.[15] The EU originated in Western Europe but has been expanding eastward since the fall of the Soviet Union in 1991. The currency of most countries of the European Union, the euro, is the most commonly used among Europeans; and the EU's Schengen Area abolishes border and immigration controls between most of its member states. There exists a political movement favoring the evolution of the European Union into a single federation encompassing much of the continent.

In classical Greek mythology, Europa (Ancient Greek: Εὐρώπη, Eurṓpē) was a Phoenician princess. One view is that her name derives from the ancient Greek elements εὐρύς (eurús), "wide, broad" and ὤψ (ōps, gen. ὠπός, ōpós) "eye, face, countenance", hence their composite Eurṓpē would mean "wide-gazing" or "broad of aspect".[16][17][18] Broad has been an epithet of Earth herself in the reconstructed Proto-Indo-European religion and the poetry devoted to it.[16] An alternative view is that of R.S.P. Beekes who has argued in favor of a Pre-Indo-European origin for the name, explaining that a derivation from ancient Greek eurus would yield a different toponym than Europa. Beekes has located toponyms related to that of Europa in the territory of ancient Greece and localities like that of Europos in ancient Macedonia.[19]

There have been attempts to connect Eurṓpē to a Semitic term for "west", this being either Akkadian erebu meaning "to go down, set" (said of the sun) or Phoenician 'ereb "evening, west",[20] which is at the origin of Arabic Maghreb and Hebrew ma'arav. Michael A. Barry finds the mention of the word Ereb on an Assyrian stele with the meaning of "night, [the country of] sunset", in opposition to Asu "[the country of] sunrise", i.e. Asia. The same naming motive according to "cartographic convention" appears in Greek Ἀνατολή (Anatolḗ "[sun] rise", "east", hence Anatolia).[21] Martin Litchfield West stated that "phonologically, the match between Europa's name and any form of the Semitic word is very poor",[22] while Beekes considers a connection to Semitic languages improbable.[19] Next to these hypotheses there is also a Proto-Indo-European root *h1regʷos, meaning "darkness", which also produced Greek Erebus.[citation needed]

Most major world languages use words derived from Eurṓpē or Europa to refer to the continent. Chinese, for example, uses the word Ōuzhōu (歐洲/欧洲), which is an abbreviation of the transliterated name Ōuluóbā zhōu (歐羅巴洲) (zhōu means "continent"); a similar Chinese-derived term Ōshū (欧州) is also sometimes used in Japanese such as in the Japanese name of the European Union, Ōshū Rengō (欧州連合), despite the katakana Yōroppa (ヨーロッパ) being more commonly used. In some Turkic languages, the originally Persian name Frangistan ("land of the Franks") is used casually in referring to much of Europe, besides official names such as Avrupa or Evropa.[23]"""

## We split this article into paragraphs and then every paragraph into sentences
paragraphs = []
for paragraph in document.replace("\r\n", "\n").split("\n\n"):
    if len(paragraph.strip()) > 0:
        paragraphs.append(sent_tokenize(paragraph.strip()))

#We combine up to 3 sentences into a passage. You can choose smaller or larger values for window_size
#Smaller value: Context from other sentences might get lost
#Lager values: More context from the paragraph remains, but results are longer
window_size = 3
passages = []
for paragraph in paragraphs:
    for start_idx in range(0, len(paragraph), window_size):
        end_idx = min(start_idx+window_size, len(paragraph))
        passages.append(" ".join(paragraph[start_idx:end_idx]))

print("Paragraphs: ", len(paragraphs))
print("Sentences: ", sum([len(p) for p in paragraphs]))
print("Passages: ", len(passages))

## Load our cross-encoder. Use fast tokenizer to speed up the tokenization
model = CrossEncoder('sentence-transformers/ce-ms-marco-TinyBERT-L-2', use_fast_tokenizer=True)

## Some queries we want to search for in the document
queries = ["How large is Europe?",
           "Is Europe a continent?",
           "What is the currency in Europe?",
           "Fall Roman Empire when",
           "Is Europe in the north or in the south?"]

#Search in a loop for the individual queries
for query in queries:
    start_time = time.time()

    #Concatenate the query and all passages and predict the scores for the pairs [query, passage]
    model_inputs = [[query, passage] for passage in passages]
    scores = model.predict(model_inputs)

    #Sort the scores in decreasing order
    results = [{'input': inp, 'score': score} for inp, score in zip(model_inputs, scores)]
    results = sorted(results, key=lambda x: x['score'], reverse=True)

    print("Query:", query)
    print("Search took {:.2f} seconds".format(time.time() - start_time))
    for hit in results[0:5]:
        print("Score: {:.2f}".format(hit['score']), "\t", hit['input'][1])

    print("==========")<commit_msg>update queries<commit_after>"""
This example show how in-document search can be used with a CrossEncoder.

The document is split into passage. Here, we use three consecutive sentences as a passage. You can use shorter passage, for example, individual sentences,
or longer passages, like full paragraphs.

The CrossEncoder takes the search query and scores every passage how relevant the passage is for the given score. The five passages with the highest score are  then returned.

As CrossEncoder, we use sentence-transformers/ce-ms-marco-TinyBERT-L-2, a BERT model with only 2 layers trained on the MS MARCO dataset. This is an extremely quick model able to score up to 9000 passages per second (on a V100 GPU). You can also use a larger model, which gives better results but is also slower.

Note: As we score the [query, passage]-pair for every new query, this search method
becomes at some point in-efficient if the document gets too large.

Usage: python in_document_search_crossencoder.py
"""

from sentence_transformers import CrossEncoder
from nltk import sent_tokenize
import time

#As document, we take the first two section from the Wikipedia article about Europe
document = """Europe is a continent located entirely in the Northern Hemisphere and mostly in the Eastern Hemisphere. It comprises the westernmost part of Eurasia and is bordered by the Arctic Ocean to the north, the Atlantic Ocean to the west, the Mediterranean Sea to the south, and Asia to the east. Europe is commonly considered to be separated from Asia by the watershed of the Ural Mountains, the Ural River, the Caspian Sea, the Greater Caucasus, the Black Sea, and the waterways of the Turkish Straits. Although some of this border is over land, Europe is generally accorded the status of a full continent because of its great physical size and the weight of history and tradition.

Europe covers about 10,180,000 square kilometres (3,930,000 sq mi), or 2% of the Earth's surface (6.8% of land area), making it the second smallest continent. Politically, Europe is divided into about fifty sovereign states, of which Russia is the largest and most populous, spanning 39% of the continent and comprising 15% of its population. Europe had a total population of about 741 million (about 11% of the world population) as of 2018. The European climate is largely affected by warm Atlantic currents that temper winters and summers on much of the continent, even at latitudes along which the climate in Asia and North America is severe. Further from the sea, seasonal differences are more noticeable than close to the coast.

European culture is the root of Western civilization, which traces its lineage back to ancient Greece and ancient Rome. The fall of the Western Roman Empire in 476 AD and the subsequent Migration Period marked the end of Europe's ancient history and the beginning of the Middle Ages. Renaissance humanism, exploration, art and science led to the modern era. Since the Age of Discovery, started by Portugal and Spain, Europe played a predominant role in global affairs. Between the 16th and 20th centuries, European powers colonized at various times the Americas, almost all of Africa and Oceania, and the majority of Asia.

The Age of Enlightenment, the subsequent French Revolution and the Napoleonic Wars shaped the continent culturally, politically and economically from the end of the 17th century until the first half of the 19th century. The Industrial Revolution, which began in Great Britain at the end of the 18th century, gave rise to radical economic, cultural and social change in Western Europe and eventually the wider world. Both world wars took place for the most part in Europe, contributing to a decline in Western European dominance in world affairs by the mid-20th century as the Soviet Union and the United States took prominence. During the Cold War, Europe was divided along the Iron Curtain between NATO in the West and the Warsaw Pact in the East, until the revolutions of 1989 and fall of the Berlin Wall.

In 1949, the Council of Europe was founded with the idea of unifying Europe to achieve common goals. Further European integration by some states led to the formation of the European Union (EU), a separate political entity that lies between a confederation and a federation. The EU originated in Western Europe but has been expanding eastward since the fall of the Soviet Union in 1991. The currency of most countries of the European Union, the euro, is the most commonly used among Europeans; and the EU's Schengen Area abolishes border and immigration controls between most of its member states. There exists a political movement favoring the evolution of the European Union into a single federation encompassing much of the continent.

In classical Greek mythology, Europa (Ancient Greek: Εὐρώπη, Eurṓpē) was a Phoenician princess. One view is that her name derives from the ancient Greek elements εὐρύς (eurús), "wide, broad" and ὤψ (ōps, gen. ὠπός, ōpós) "eye, face, countenance", hence their composite Eurṓpē would mean "wide-gazing" or "broad of aspect". Broad has been an epithet of Earth herself in the reconstructed Proto-Indo-European religion and the poetry devoted to it. An alternative view is that of R.S.P. Beekes who has argued in favor of a Pre-Indo-European origin for the name, explaining that a derivation from ancient Greek eurus would yield a different toponym than Europa. Beekes has located toponyms related to that of Europa in the territory of ancient Greece and localities like that of Europos in ancient Macedonia.

There have been attempts to connect Eurṓpē to a Semitic term for "west", this being either Akkadian erebu meaning "to go down, set" (said of the sun) or Phoenician 'ereb "evening, west", which is at the origin of Arabic Maghreb and Hebrew ma'arav. Michael A. Barry finds the mention of the word Ereb on an Assyrian stele with the meaning of "night, [the country of] sunset", in opposition to Asu "[the country of] sunrise", i.e. Asia. The same naming motive according to "cartographic convention" appears in Greek Ἀνατολή (Anatolḗ "[sun] rise", "east", hence Anatolia). Martin Litchfield West stated that "phonologically, the match between Europa's name and any form of the Semitic word is very poor", while Beekes considers a connection to Semitic languages improbable. Next to these hypotheses there is also a Proto-Indo-European root *h1regʷos, meaning "darkness", which also produced Greek Erebus.

Most major world languages use words derived from Eurṓpē or Europa to refer to the continent. Chinese, for example, uses the word Ōuzhōu (歐洲/欧洲), which is an abbreviation of the transliterated name Ōuluóbā zhōu (歐羅巴洲) (zhōu means "continent"); a similar Chinese-derived term Ōshū (欧州) is also sometimes used in Japanese such as in the Japanese name of the European Union, Ōshū Rengō (欧州連合), despite the katakana Yōroppa (ヨーロッパ) being more commonly used. In some Turkic languages, the originally Persian name Frangistan ("land of the Franks") is used casually in referring to much of Europe, besides official names such as Avrupa or Evropa."""

## We split this article into paragraphs and then every paragraph into sentences
paragraphs = []
for paragraph in document.replace("\r\n", "\n").split("\n\n"):
    if len(paragraph.strip()) > 0:
        paragraphs.append(sent_tokenize(paragraph.strip()))

#We combine up to 3 sentences into a passage. You can choose smaller or larger values for window_size
#Smaller value: Context from other sentences might get lost
#Lager values: More context from the paragraph remains, but results are longer
window_size = 3
passages = []
for paragraph in paragraphs:
    for start_idx in range(0, len(paragraph), window_size):
        end_idx = min(start_idx+window_size, len(paragraph))
        passages.append(" ".join(paragraph[start_idx:end_idx]))

print("Paragraphs: ", len(paragraphs))
print("Sentences: ", sum([len(p) for p in paragraphs]))
print("Passages: ", len(passages))

## Load our cross-encoder. Use fast tokenizer to speed up the tokenization
model = CrossEncoder('sentence-transformers/ce-ms-marco-TinyBERT-L-2', use_fast_tokenizer=True)

## Some queries we want to search for in the document
queries = ["How large is Europe?",
           "Is Europe a continent?",
           "What is the currency in EU?",
           "Fall Roman Empire when",                    #We can also search for key word queries
           "Is Europa in the south part of the globe?"]   #Europe is miss-spelled & the matching sentences does not mention any of the content words

#Search in a loop for the individual queries
for query in queries:
    start_time = time.time()

    #Concatenate the query and all passages and predict the scores for the pairs [query, passage]
    model_inputs = [[query, passage] for passage in passages]
    scores = model.predict(model_inputs)

    #Sort the scores in decreasing order
    results = [{'input': inp, 'score': score} for inp, score in zip(model_inputs, scores)]
    results = sorted(results, key=lambda x: x['score'], reverse=True)

    print("Query:", query)
    print("Search took {:.2f} seconds".format(time.time() - start_time))
    for hit in results[0:5]:
        print("Score: {:.2f}".format(hit['score']), "\t", hit['input'][1])

    print("==========")<|endoftext|>


<commit_before># Copyright 2022 Tecnativa - Carlos Roca
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
    "name": "Account Sale Stock Report Non Billed",
    "version": "13.0.1.0.1",
    "license": "AGPL-3",
    "author": "Tecnativa, Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/account-financial-reporting",
    "depends": ["stock_picking_invoice_link"],
    "data": [
        "views/res_config_settings_views.xml",
        "views/stock_move_non_billed_views.xml",
        "wizard/account_sale_stock_report_non_billed_wiz_views.xml",
    ],
    "installable": True,
    "maintainers": ["CarlosRoca13"],
    "development_status": "Beta",
}
<commit_msg>account_sale_stock_report_non_billed 13.0.1.1.0<commit_after># Copyright 2022 Tecnativa - Carlos Roca
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
    "name": "Account Sale Stock Report Non Billed",
    "version": "13.0.1.1.0",
    "license": "AGPL-3",
    "author": "Tecnativa, Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/account-financial-reporting",
    "depends": ["stock_picking_invoice_link"],
    "data": [
        "views/res_config_settings_views.xml",
        "views/stock_move_non_billed_views.xml",
        "wizard/account_sale_stock_report_non_billed_wiz_views.xml",
    ],
    "installable": True,
    "maintainers": ["CarlosRoca13"],
    "development_status": "Beta",
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, api
import logging
_logger = logging.getLogger(__name__)

class WizardMultiChartsAccounts(models.TransientModel):
    _inherit = 'wizard.multi.charts.accounts'

    @api.multi
    def _create_bank_journals_from_o2m(self, company, acc_template_ref):
        """
        Bank - Cash journals are created with this method
        Inherit this function in order to add checks to cash and bank
        journals. This is because usually will be installed before chart loaded
        and they will be disable by default
        """
        res = super(
            WizardMultiChartsAccounts, self)._create_bank_journals_from_o2m(
            company, acc_template_ref)

        # creamos diario para retenciones
        inbound_withholding = self.env.ref(
            'account_withholding.account_payment_method_in_withholding')
        outbound_withholding = self.env.ref(
            'account_withholding.account_payment_method_out_withholding')
        journal = self.env['account.journal'].create({
            'name': 'Retenciones',
            'type': 'cash',
            'company_id': company.id,
            'inbound_payment_method_ids': [
                (4, inbound_withholding.id, None)],
            'outbound_payment_method_ids': [
                (4, outbound_withholding.id, None)],
        })
        # we dont want this journal to have accounts and we can not inherit
        # to avoid creation, so we delete it
        journal.default_credit_account_id.unlink()

        return res
<commit_msg>IMP only install withholding journal if localization whant it<commit_after># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, api
import logging
_logger = logging.getLogger(__name__)

class WizardMultiChartsAccounts(models.TransientModel):
    _inherit = 'wizard.multi.charts.accounts'

    @api.multi
    def _create_bank_journals_from_o2m(self, company, acc_template_ref):
        """
        Bank - Cash journals are created with this method
        Inherit this function in order to add checks to cash and bank
        journals. This is because usually will be installed before chart loaded
        and they will be disable by default
        """
        res = super(
            WizardMultiChartsAccounts, self)._create_bank_journals_from_o2m(
            company, acc_template_ref)

        # each chart of account / localization should send this key if
        # they want withholding journal to be created
        if self._context.get('create_withholding_journal'):
            # creamos diario para retenciones
            inbound_withholding = self.env.ref(
                'account_withholding.account_payment_method_in_withholding')
            outbound_withholding = self.env.ref(
                'account_withholding.account_payment_method_out_withholding')
            journal = self.env['account.journal'].create({
                'name': 'Retenciones',
                'type': 'cash',
                'company_id': company.id,
                'inbound_payment_method_ids': [
                    (4, inbound_withholding.id, None)],
                'outbound_payment_method_ids': [
                    (4, outbound_withholding.id, None)],
            })
            # we dont want this journal to have accounts and we can not inherit
            # to avoid creation, so we delete it
            journal.default_credit_account_id.unlink()

        return res
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import osv
from tools.translate import _
import re
try:
    from mygengo import MyGengo
except ImportError:
    raise osv.except_osv(_('Gengo ImportError'), _('Please install mygengo lib from http://pypi.python.org/pypi/mygengo'))

import logging
import tools
import time

_logger = logging.getLogger(__name__)

GENGO_DEFAULT_LIMIT = 20

LANG_CODE_MAPPING = {
    'ar_SA': ('ar', 'Arabic'),
    'id_ID': ('id', 'Indonesian'),
    'nl_NL': ('nl', 'Dutch'),
    'fr_CA': ('fr-ca', 'French (Canada)'),
    'pl_PL': ('pl', 'Polish'),
    'zh_TW': ('zh-tw', 'Chinese (Traditional)'),
    'sv_SE': ('sv', 'Swedish'),
    'ko_KR': ('ko', 'Korean'),
    'pt_PT': ('pt', 'Portuguese (Europe)'),
    'en_US': ('en', 'English'),
    'ja_JP': ('ja', 'Japanese'),
    'es_ES': ('es', 'Spanish (Spain)'),
    'zh_CN': ('zh', 'Chinese (Simplified)'),
    'de_DE': ('de', 'German'),
    'fr_FR': ('fr', 'French'),
    'fr_BE': ('fr', 'French'),
    'ru_RU': ('ru', 'Russian'),
    'it_IT': ('it', 'Italian'),
    'pt_BR': ('pt-br', 'Portuguese (Brazil)')
}

CRON_VALS = {
    'name': _('Synchronization with Gengo'),
    'active': True,
    'interval_number': 20,
    'interval_type': 'minutes',
    'numbercall': -1,
    'model': "'base.update.translations'",
    'function': "",
    'args': "'(%s,)'" %s GENGO_DEFAULT_LIMIT,
}

class base_update_translation(osv.osv_memory):

    _name = 'base.update.translations'
    _inherit = "base.update.translations"

    def gengo_authentication(self, cr, uid, context=None):
        ''' To Send Request and Get Response from Gengo User needs Public and Private
         key for that user need to sign up to gengo and get public and private
         key which is provided by gengo to authenticate user '''

        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        if not user.company_id.gengo_public_key or not user.company_id.gengo_private_key:
            return (False, _("  - Invalid Gengo configuration. Gengo authentication `Public Key` or `Private Key` is missing. Complete Gengo authentication parameters under `Settings > Companies > Gengo Parameters`."))
        try:
            gengo = MyGengo(
                public_key=user.company_id.gengo_public_key.encode('ascii'),
                private_key=user.company_id.gengo_private_key.encode('ascii'),
                sandbox=True,
            )
            gengo.getAccountStats()
            return (True, gengo)
        except Exception, e:
            return (False, _("Gengo Connection Error\n%s") %e)

    def pack_jobs_request(self, cr, uid, term_ids, context=None):
        ''' prepare the terms that will be requested to gengo and returns them in a dictionary with following format
            {'jobs': {
                'term1.id': {...}
                'term2.id': {...}
                }
            }'''

        translation_pool = self.pool.get('ir.translation')
        jobs = {}
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        auto_approve = 1 if user.company_id.gengo_auto_approve else 0
        for term in translation_pool.browse(cr, uid, term_ids, context=context):
            if re.search(r"\w", term.src or ""):
                jobs[term.id] = {'type': 'text',
                        'slug': 'single::English to ' + LANG_CODE_MAPPING[term.lang][1],
                        'tier': tools.ustr(user.company_id.gengo_tier),
                        #'tier': tools.ustr(term.gengo_tier),
                        'body_src': term.src,
                        'lc_src': 'en',
                        'lc_tgt': LANG_CODE_MAPPING[term.lang][0],
                        'auto_approve': auto_approve,
                        'comment': gengo_parameter_pool.company_id.gengo_comment,
                }
        return {'jobs': jobs}

    def check_lang_support(self, cr, uid, langs, context=None):
        new_langs = []
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            return []
        else:
            user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
            tier = user.company_id.gengo_tier
            if tier == "machine":
                tier = "nonprofit"
            lang_pair = gengo.getServiceLanguagePairs(lc_src='en')
            #print "tier", tier

            if lang_pair['opstat'] == 'ok':
                for g_lang in lang_pair['response']:
                     #print 'g_lang', g_lang['lc_tgt'], g_lang['tier']
                    for l in langs:
                        if LANG_CODE_MAPPING[l][0] == g_lang['lc_tgt'] and g_lang['tier'] == tier:
                            new_langs.append(l)
            return list(set(new_langs))

    def _update_terms(self, cr, uid, response, tier, context=None):
        translation_pool = self.pool.get('ir.translation')
        for jobs in response['jobs']:
            for t_id, res in jobs.items():
                vals = {}
                if tier == "machine":
                    vals.update({'value': res['body_tgt'], 'state': 'translated'})
                else:
                    vals.update({'job_id': res['job_id'], 'state': 'inprogress'})
                translation_pool.write(cr, uid, [t_id], vals, context=context)
        return

    def _send_translation_terms(self, cr, uid,  term_ids, context=None):
        """
        Lazy Polling will be perform when user or cron request for the translation.
        """
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if flag:
            request = self.pack_jobs_request(cr, uid, term_ids, context)
            if request['jobs']:
                result = gengo.postTranslationJobs(jobs=request)
                if result['opstat'] == 'ok':
                    self._update_terms(cr, uid, result['response'], user.company_id.gengo_tier, context)
        else:
            _logger.error(gengo)
        return True

    def do_check_schedular(self, cr, uid, xml_id, name, fn, context=None):
        cron_pool = self.pool.get('ir.cron')
        try:
            res = []
            model, res = self.pool.get('ir.model.data').get_object_reference(cr, uid, 'base_gengo', xml_id)
            cron_pool.write(cr, uid, [res], {'active': True}, context=context)
        except:
            #the cron job was not found, probably deleted previously, so we create it again using default values
            CRON_VALS.update({'name': name, "function": fn})
            return cron_pool.create(cr, uid, CRON_VALS, context)

    def act_update(self, cr, uid, ids, context=None):
        if context == None:
            context = {}
        lang_pool = self.pool.get('res.lang')
        super(base_update_translation, self).act_update(cr, uid, ids, context)
        msg = _("1. Translation file loaded successfully.\n2. Processing Gengo Translation:\n")
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            msg += gengo
        else:
            for res in self.browse(cr, uid, ids, context=context):
                lang_id = lang_pool.search(cr, uid, [('code', '=', res.lang)])
                lang_name = self._get_lang_name(cr, uid, res.lang)
                try:
                    if LANG_CODE_MAPPING[res.lang][0]:
                        lang_search = lang_pool.search(cr, uid, [('gengo_sync', '=', True), ('id', '=', lang_id[0])])
                    if lang_search:
                        msg += _('  - This language `%s` is already in queue for translation.') % (lang_name)
                    else:
                        msg += _('  - The language `%s` is queued for translation through Gengo translation.') % (lang_name)
                        lang_pool.write(cr, uid, lang_id, {'gengo_sync': True})
                        _logger.info('Translation request for language `%s` has been queued successfully.', lang_name)
                    self.do_check_schedular(cr, uid, 'gengo_sync_send_request_scheduler', _('Gengo Sync Translation (Request)'), '_sync_request', context)
                    self.do_check_schedular(cr, uid, 'gengo_sync_receive_request_scheduler', _('Gengo Sync Translation (Response)'), '_sync_response', context)
                    self._sync_request(cr, uid, limit=GENGO_DEFAULT_LIMIT)
                except:
                    msg += _('  - The Language `%s` is not supported by Gengo Traditional Service.') % (lang_name)

        obj_model = self.pool.get('ir.model.data')
        model_data_ids = obj_model.search(cr, uid, [('model', '=', 'ir.ui.view'), ('name', '=', 'update_translation_wizard_view_confirm')])
        view_id = obj_model.read(cr, uid, model_data_ids, fields=['res_id'], context=context)[0]['res_id']
        context.update({'message': msg})
        return {
                 'view_type': 'form',
                 'view_mode': 'form',
                 'res_model': 'gengo.update.message',
                 'views': [(view_id, 'form')],
                 'type': 'ir.actions.act_window',
                 'target': 'new',
                 'context': context,
             }

    def _sync_response(self, cr, uid, limit=GENGO_DEFAULT_LIMIT, context=None):
        """
        This method  will be call by cron services to get translation from
        gengo for translation terms which are posted to be translated. It will
        read translated terms and comments from gengo and will update respective
        translation in openerp.
        """
        translation_pool = self.pool.get('ir.translation')
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            _logger.warning("%s", gengo)
        else:
            translation_id = translation_pool.search(cr, uid, [('state', '=', 'inprogress'), ('gengo_translation', '=', True)], limit=limit, context=context)
            for term in translation_pool.browse(cr, uid, translation_id, context):
                up_term = up_comment = 0
                if term.job_id:
                    vals={}
                    job_response = gengo.getTranslationJob(id=term.job_id)
                    if job_response['opstat'] != 'ok':
                        _logger.warning("Invalid Response! Skipping translation Terms with `id` %s." % (term.job_id))
                        continue
                    if job_response['response']['job']['status'] == 'approved':
                        vals.update({'state': 'translated',
                            'value': job_response['response']['job']['body_tgt'])
                        up_term += 1
                    job_comment = gengo.getTranslationJobComments(id=term.job_id)
                    if job_comment['opstat']=='ok':
                        gengo_comments=""
                        for comment in job_comment['response']['thread']:
                            gengo_comments += _('%s Commented on %s by %s. \n') %(comment['body'], time.ctime(comment['ctime']), comment['author'])
                        vals.update({'gengo_comment': gengo_comments})
                        up_comment +=1
                    if vals:
                        translation_pool.write(cr, uid, term.id,vals)
                    _logger.info("Successfully Updated `%d` terms and Comments for `%d` terms." % (up_term, up_comment ))
        return True

    def _sync_request(self, cr, uid, limit=GENGO_DEFAULT_LIMIT, context=None):
        """This scheduler will send a job request to the gengo , which terms are
        waiing to be translated and for which gengo_translation is True"""
        if context is None:
            context = {}
        language_pool = self.pool.get('res.lang')
        translation_pool = self.pool.get('ir.translation')
        try:
            lang_ids = language_pool.search(cr, uid, [('gengo_sync', '=', True)]) #really? what's the point not checking ALL the languages if gengo support it? i feel like it's aunecessary load for the user that must configure yet another thing
            langs = [lang.code for lang in language_pool.browse(cr, uid, lang_ids, context=context)]
            #print "LANGS 1", langs
            langs = self.check_lang_support(cr, uid, langs, context=context)#must move
            #print "LANGS 2", langs
            term_ids = translation_pool.search(cr, uid, [('state', '=', 'to_translate'), ('gengo_translation', '=', True), ('lang', 'in', langs)], limit=limit)
            if term_ids:
                self._send_translation_terms(cr, uid, term_ids, context)
                _logger.info("Translation terms %s has been posted to gengo successfully", len(term_ids))
            else:
                _logger.info('No Translation terms to process.')
        except Exception, e:
            _logger.error("%s", e)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX] base_gengo: syntax error fixed<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import osv
from tools.translate import _
import re
try:
    from mygengo import MyGengo
except ImportError:
    raise osv.except_osv(_('Gengo ImportError'), _('Please install mygengo lib from http://pypi.python.org/pypi/mygengo'))

import logging
import tools
import time

_logger = logging.getLogger(__name__)

GENGO_DEFAULT_LIMIT = 20

LANG_CODE_MAPPING = {
    'ar_SA': ('ar', 'Arabic'),
    'id_ID': ('id', 'Indonesian'),
    'nl_NL': ('nl', 'Dutch'),
    'fr_CA': ('fr-ca', 'French (Canada)'),
    'pl_PL': ('pl', 'Polish'),
    'zh_TW': ('zh-tw', 'Chinese (Traditional)'),
    'sv_SE': ('sv', 'Swedish'),
    'ko_KR': ('ko', 'Korean'),
    'pt_PT': ('pt', 'Portuguese (Europe)'),
    'en_US': ('en', 'English'),
    'ja_JP': ('ja', 'Japanese'),
    'es_ES': ('es', 'Spanish (Spain)'),
    'zh_CN': ('zh', 'Chinese (Simplified)'),
    'de_DE': ('de', 'German'),
    'fr_FR': ('fr', 'French'),
    'fr_BE': ('fr', 'French'),
    'ru_RU': ('ru', 'Russian'),
    'it_IT': ('it', 'Italian'),
    'pt_BR': ('pt-br', 'Portuguese (Brazil)')
}

CRON_VALS = {
    'name': _('Synchronization with Gengo'),
    'active': True,
    'interval_number': 20,
    'interval_type': 'minutes',
    'numbercall': -1,
    'model': "'base.update.translations'",
    'function': "",
    'args': "'(%s,)'" % (str(GENGO_DEFAULT_LIMIT)),
}

class base_update_translation(osv.osv_memory):

    _name = 'base.update.translations'
    _inherit = "base.update.translations"

    def gengo_authentication(self, cr, uid, context=None):
        ''' To Send Request and Get Response from Gengo User needs Public and Private
         key for that user need to sign up to gengo and get public and private
         key which is provided by gengo to authenticate user '''

        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        if not user.company_id.gengo_public_key or not user.company_id.gengo_private_key:
            return (False, _("  - Invalid Gengo configuration. Gengo authentication `Public Key` or `Private Key` is missing. Complete Gengo authentication parameters under `Settings > Companies > Gengo Parameters`."))
        try:
            gengo = MyGengo(
                public_key=user.company_id.gengo_public_key.encode('ascii'),
                private_key=user.company_id.gengo_private_key.encode('ascii'),
                sandbox=True,
            )
            gengo.getAccountStats()
            return (True, gengo)
        except Exception, e:
            return (False, _("Gengo Connection Error\n%s") %e)

    def pack_jobs_request(self, cr, uid, term_ids, context=None):
        ''' prepare the terms that will be requested to gengo and returns them in a dictionary with following format
            {'jobs': {
                'term1.id': {...}
                'term2.id': {...}
                }
            }'''

        translation_pool = self.pool.get('ir.translation')
        jobs = {}
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        auto_approve = 1 if user.company_id.gengo_auto_approve else 0
        for term in translation_pool.browse(cr, uid, term_ids, context=context):
            if re.search(r"\w", term.src or ""):
                jobs[term.id] = {'type': 'text',
                        'slug': 'single::English to ' + LANG_CODE_MAPPING[term.lang][1],
                        'tier': tools.ustr(user.company_id.gengo_tier),
                        #'tier': tools.ustr(term.gengo_tier),
                        'body_src': term.src,
                        'lc_src': 'en',
                        'lc_tgt': LANG_CODE_MAPPING[term.lang][0],
                        'auto_approve': auto_approve,
                        'comment': gengo_parameter_pool.company_id.gengo_comment,
                }
        return {'jobs': jobs}

    def check_lang_support(self, cr, uid, langs, context=None):
        new_langs = []
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            return []
        else:
            user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
            tier = user.company_id.gengo_tier
            if tier == "machine":
                tier = "nonprofit"
            lang_pair = gengo.getServiceLanguagePairs(lc_src='en')
            #print "tier", tier

            if lang_pair['opstat'] == 'ok':
                for g_lang in lang_pair['response']:
                     #print 'g_lang', g_lang['lc_tgt'], g_lang['tier']
                    for l in langs:
                        if LANG_CODE_MAPPING[l][0] == g_lang['lc_tgt'] and g_lang['tier'] == tier:
                            new_langs.append(l)
            return list(set(new_langs))

    def _update_terms(self, cr, uid, response, tier, context=None):
        translation_pool = self.pool.get('ir.translation')
        for jobs in response['jobs']:
            for t_id, res in jobs.items():
                vals = {}
                if tier == "machine":
                    vals.update({'value': res['body_tgt'], 'state': 'translated'})
                else:
                    vals.update({'job_id': res['job_id'], 'state': 'inprogress'})
                translation_pool.write(cr, uid, [t_id], vals, context=context)
        return

    def _send_translation_terms(self, cr, uid,  term_ids, context=None):
        """
        Lazy Polling will be perform when user or cron request for the translation.
        """
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if flag:
            request = self.pack_jobs_request(cr, uid, term_ids, context)
            if request['jobs']:
                result = gengo.postTranslationJobs(jobs=request)
                if result['opstat'] == 'ok':
                    self._update_terms(cr, uid, result['response'], user.company_id.gengo_tier, context)
        else:
            _logger.error(gengo)
        return True

    def do_check_schedular(self, cr, uid, xml_id, name, fn, context=None):
        cron_pool = self.pool.get('ir.cron')
        try:
            res = []
            model, res = self.pool.get('ir.model.data').get_object_reference(cr, uid, 'base_gengo', xml_id)
            cron_pool.write(cr, uid, [res], {'active': True}, context=context)
        except:
            #the cron job was not found, probably deleted previously, so we create it again using default values
            CRON_VALS.update({'name': name, "function": fn})
            return cron_pool.create(cr, uid, CRON_VALS, context)

    def act_update(self, cr, uid, ids, context=None):
        if context == None:
            context = {}
        lang_pool = self.pool.get('res.lang')
        super(base_update_translation, self).act_update(cr, uid, ids, context)
        msg = _("1. Translation file loaded successfully.\n2. Processing Gengo Translation:\n")
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            msg += gengo
        else:
            for res in self.browse(cr, uid, ids, context=context):
                lang_id = lang_pool.search(cr, uid, [('code', '=', res.lang)])
                lang_name = self._get_lang_name(cr, uid, res.lang)
                try:
                    if LANG_CODE_MAPPING[res.lang][0]:
                        lang_search = lang_pool.search(cr, uid, [('gengo_sync', '=', True), ('id', '=', lang_id[0])])
                    if lang_search:
                        msg += _('  - This language `%s` is already in queue for translation.') % (lang_name)
                    else:
                        msg += _('  - The language `%s` is queued for translation through Gengo translation.') % (lang_name)
                        lang_pool.write(cr, uid, lang_id, {'gengo_sync': True})
                        _logger.info('Translation request for language `%s` has been queued successfully.', lang_name)
                    self.do_check_schedular(cr, uid, 'gengo_sync_send_request_scheduler', _('Gengo Sync Translation (Request)'), '_sync_request', context)
                    self.do_check_schedular(cr, uid, 'gengo_sync_receive_request_scheduler', _('Gengo Sync Translation (Response)'), '_sync_response', context)
                    self._sync_request(cr, uid, limit=GENGO_DEFAULT_LIMIT)
                except:
                    msg += _('  - The Language `%s` is not supported by Gengo Traditional Service.') % (lang_name)

        obj_model = self.pool.get('ir.model.data')
        model_data_ids = obj_model.search(cr, uid, [('model', '=', 'ir.ui.view'), ('name', '=', 'update_translation_wizard_view_confirm')])
        view_id = obj_model.read(cr, uid, model_data_ids, fields=['res_id'], context=context)[0]['res_id']
        context.update({'message': msg})
        return {
                 'view_type': 'form',
                 'view_mode': 'form',
                 'res_model': 'gengo.update.message',
                 'views': [(view_id, 'form')],
                 'type': 'ir.actions.act_window',
                 'target': 'new',
                 'context': context,
             }

    def _sync_response(self, cr, uid, limit=GENGO_DEFAULT_LIMIT, context=None):
        """
        This method  will be call by cron services to get translation from
        gengo for translation terms which are posted to be translated. It will
        read translated terms and comments from gengo and will update respective
        translation in openerp.
        """
        translation_pool = self.pool.get('ir.translation')
        flag, gengo = self.gengo_authentication(cr, uid, context)
        if not flag:
            _logger.warning("%s", gengo)
        else:
            translation_id = translation_pool.search(cr, uid, [('state', '=', 'inprogress'), ('gengo_translation', '=', True)], limit=limit, context=context)
            for term in translation_pool.browse(cr, uid, translation_id, context):
                up_term = up_comment = 0
                if term.job_id:
                    vals={}
                    job_response = gengo.getTranslationJob(id=term.job_id)
                    if job_response['opstat'] != 'ok':
                        _logger.warning("Invalid Response! Skipping translation Terms with `id` %s." % (term.job_id))
                        continue
                    if job_response['response']['job']['status'] == 'approved':
                        vals.update({'state': 'translated',
                            'value': job_response['response']['job']['body_tgt']})
                        up_term += 1
                    job_comment = gengo.getTranslationJobComments(id=term.job_id)
                    if job_comment['opstat']=='ok':
                        gengo_comments=""
                        for comment in job_comment['response']['thread']:
                            gengo_comments += _('%s Commented on %s by %s. \n') %(comment['body'], time.ctime(comment['ctime']), comment['author'])
                        vals.update({'gengo_comment': gengo_comments})
                        up_comment +=1
                    if vals:
                        translation_pool.write(cr, uid, term.id,vals)
                    _logger.info("Successfully Updated `%d` terms and Comments for `%d` terms." % (up_term, up_comment ))
        return True

    def _sync_request(self, cr, uid, limit=GENGO_DEFAULT_LIMIT, context=None):
        """This scheduler will send a job request to the gengo , which terms are
        waiing to be translated and for which gengo_translation is True"""
        if context is None:
            context = {}
        language_pool = self.pool.get('res.lang')
        translation_pool = self.pool.get('ir.translation')
        try:
            lang_ids = language_pool.search(cr, uid, [('gengo_sync', '=', True)]) #really? what's the point not checking ALL the languages if gengo support it? i feel like it's aunecessary load for the user that must configure yet another thing
            langs = [lang.code for lang in language_pool.browse(cr, uid, lang_ids, context=context)]
            #print "LANGS 1", langs
            langs = self.check_lang_support(cr, uid, langs, context=context)#must move
            #print "LANGS 2", langs
            term_ids = translation_pool.search(cr, uid, [('state', '=', 'to_translate'), ('gengo_translation', '=', True), ('lang', 'in', langs)], limit=limit)
            if term_ids:
                self._send_translation_terms(cr, uid, term_ids, context)
                _logger.info("Translation terms %s has been posted to gengo successfully", len(term_ids))
            else:
                _logger.info('No Translation terms to process.')
        except Exception, e:
            _logger.error("%s", e)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2010-Today OpenERP SA (<http://www.openerp.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from openerp import tools
from openerp.osv import osv, fields

def _reopen(self, res_id, model):
    return {'type': 'ir.actions.act_window',
            'view_mode': 'form',
            'view_type': 'form',
            'res_id': res_id,
            'res_model': self._name,
            'target': 'new',
            # save original model in context, because selecting the list of available
            # templates requires a model in context
            'context': {
                'default_model': model,
            },
    }

class mail_compose_message(osv.TransientModel):
    _inherit = 'mail.compose.message'

    def _get_templates(self, cr, uid, context=None):
        if context is None:
            context = {}
        model = False
        email_template_obj = self.pool.get('email.template')
        message_id = context.get('default_parent_id', context.get('message_id', context.get('active_id')))

        if context.get('default_composition_mode') == 'reply' and message_id:
            message_data = self.pool.get('mail.message').browse(cr, uid, message_id, context=context)
            if message_data:
                model = message_data.model
        else:
            model = context.get('default_model', context.get('active_model'))

        record_ids = email_template_obj.search(cr, uid, [('model', '=', model)], context=context)
        return email_template_obj.name_get(cr, uid, record_ids, context) + [(False, '')]

    _columns = {
        # incredible hack of the day: size=-1 means we want an int db column instead of an str one
        'template_id': fields.selection(_get_templates, 'Template', size=-1),
    }

    def onchange_template_id(self, cr, uid, ids, template_id, composition_mode, model, res_id, context=None):
        """ - mass_mailing: we cannot render, so return the template values
            - normal mode: return rendered values """
        if template_id and composition_mode == 'mass_mail':
            values = self.pool.get('email.template').read(cr, uid, template_id, ['subject', 'body_html'], context)
            values.pop('id')
        elif template_id:
            # FIXME odo: change the mail generation to avoid attachment duplication
            values = self.generate_email_for_composer(cr, uid, template_id, res_id, context=context)
            # transform attachments into attachment_ids
            values['attachment_ids'] = []
            ir_attach_obj = self.pool.get('ir.attachment')
            for attach_fname, attach_datas in values.pop('attachments', []):
                data_attach = {
                    'name': attach_fname,
                    'datas': attach_datas,
                    'datas_fname': attach_fname,
                    'res_model': model,
                    'res_id': res_id,
                    'type': 'binary', # override default_type from context, possibly meant for another model!
                }
                values['attachment_ids'].append(ir_attach_obj.create(cr, uid, data_attach, context=context))
        else:
            values = self.default_get(cr, uid, ['body', 'subject', 'partner_ids', 'attachment_ids'], context=context)

        if values.get('body_html'):
            values['body'] = values.pop('body_html')
        return {'value': values}

    def save_as_template(self, cr, uid, ids, context=None):
        """ hit save as template button: current form value will be a new
            template attached to the current document. """
        email_template = self.pool.get('email.template')
        ir_model_pool = self.pool.get('ir.model')
        for record in self.browse(cr, uid, ids, context=context):
            model_ids = ir_model_pool.search(cr, uid, [('model', '=', record.model)], context=context)
            model_id = model_ids and model_ids[0] or False
            model_name = ''
            if model_id:
                model_name = ir_model_pool.browse(cr, uid, model_id, context=context).name
            template_name = "%s: %s" % (model_name, tools.ustr(record.subject))
            values = {
                'name': template_name,
                'subject': record.subject or False,
                'body_html': record.body or False,
                'model_id': model_id or False,
                'attachment_ids': [(6, 0, [att.id for att in record.attachment_ids])]
            }
            template_id = email_template.create(cr, uid, values, context=context)
            record.write(record.onchange_template_id(template_id, record.composition_mode, record.model, record.res_id)['value'])
            return _reopen(self, record.id, record.model)

    #------------------------------------------------------
    # Wizard validation and send
    #------------------------------------------------------

    def generate_email_for_composer(self, cr, uid, template_id, res_id, context=None):
        """ Call email_template.generate_email(), get fields relevant for
            mail.compose.message, transform email_cc and email_to into partner_ids """
        template_values = self.pool.get('email.template').generate_email(cr, uid, template_id, res_id, context=context)
        # filter template values
        fields = ['body_html', 'subject', 'email_to', 'email_recipients', 'email_cc', 'attachments']
        values = dict((field, template_values[field]) for field in fields if template_values.get(field))
        values['body'] = values.pop('body_html', '')
        # transform email_to, email_cc into partner_ids
        values['partner_ids'] = []

        mails = tools.email_split(values.pop('email_to', '') + ' ' + values.pop('email_cc', ''))
        for mail in mails:
            partner_id = self.pool.get('res.partner').find_or_create(cr, uid, mail, context=context)
            values['partner_ids'].append(partner_id)
        email_recipients = values.pop('email_recipients', '')
        if email_recipients:
            for partner_id in email_recipients.split(','):
                values['partner_ids'].append(int(partner_id))

        values['partner_ids'] = list(set(values['partner_ids']))

        return values

    def render_message(self, cr, uid, wizard, res_id, context=None):
        """ Override to handle templates. """
        # generate the composer email
        if wizard.template_id:
            values = self.generate_email_for_composer(cr, uid, wizard.template_id, res_id, context=context)
        else:
            values = {}
        # get values to return
        email_dict = super(mail_compose_message, self).render_message(cr, uid, wizard, res_id, context)
        values.update(email_dict)
        return values

    def render_template(self, cr, uid, template, model, res_id, context=None):
        return self.pool.get('email.template').render_template(cr, uid, template, model, res_id, context=context)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX] email_template: void value in partner_ids when generating tmeplate does not erase wizard value anymore.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2010-Today OpenERP SA (<http://www.openerp.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from openerp import tools
from openerp.osv import osv, fields

def _reopen(self, res_id, model):
    return {'type': 'ir.actions.act_window',
            'view_mode': 'form',
            'view_type': 'form',
            'res_id': res_id,
            'res_model': self._name,
            'target': 'new',
            # save original model in context, because selecting the list of available
            # templates requires a model in context
            'context': {
                'default_model': model,
            },
    }

class mail_compose_message(osv.TransientModel):
    _inherit = 'mail.compose.message'

    def _get_templates(self, cr, uid, context=None):
        if context is None:
            context = {}
        model = False
        email_template_obj = self.pool.get('email.template')
        message_id = context.get('default_parent_id', context.get('message_id', context.get('active_id')))

        if context.get('default_composition_mode') == 'reply' and message_id:
            message_data = self.pool.get('mail.message').browse(cr, uid, message_id, context=context)
            if message_data:
                model = message_data.model
        else:
            model = context.get('default_model', context.get('active_model'))

        record_ids = email_template_obj.search(cr, uid, [('model', '=', model)], context=context)
        return email_template_obj.name_get(cr, uid, record_ids, context) + [(False, '')]

    _columns = {
        # incredible hack of the day: size=-1 means we want an int db column instead of an str one
        'template_id': fields.selection(_get_templates, 'Template', size=-1),
    }

    def onchange_template_id(self, cr, uid, ids, template_id, composition_mode, model, res_id, context=None):
        """ - mass_mailing: we cannot render, so return the template values
            - normal mode: return rendered values """
        if template_id and composition_mode == 'mass_mail':
            values = self.pool.get('email.template').read(cr, uid, template_id, ['subject', 'body_html'], context)
            values.pop('id')
        elif template_id:
            # FIXME odo: change the mail generation to avoid attachment duplication
            values = self.generate_email_for_composer(cr, uid, template_id, res_id, context=context)
            print values.get('partner_ids')
            # transform attachments into attachment_ids
            values['attachment_ids'] = []
            ir_attach_obj = self.pool.get('ir.attachment')
            for attach_fname, attach_datas in values.pop('attachments', []):
                data_attach = {
                    'name': attach_fname,
                    'datas': attach_datas,
                    'datas_fname': attach_fname,
                    'res_model': model,
                    'res_id': res_id,
                    'type': 'binary',  # override default_type from context, possibly meant for another model!
                }
                values['attachment_ids'].append(ir_attach_obj.create(cr, uid, data_attach, context=context))
        else:
            values = self.default_get(cr, uid, ['body', 'subject', 'partner_ids', 'attachment_ids'], context=context)

        if values.get('body_html'):
            values['body'] = values.pop('body_html')
        return {'value': values}

    def save_as_template(self, cr, uid, ids, context=None):
        """ hit save as template button: current form value will be a new
            template attached to the current document. """
        email_template = self.pool.get('email.template')
        ir_model_pool = self.pool.get('ir.model')
        for record in self.browse(cr, uid, ids, context=context):
            model_ids = ir_model_pool.search(cr, uid, [('model', '=', record.model)], context=context)
            model_id = model_ids and model_ids[0] or False
            model_name = ''
            if model_id:
                model_name = ir_model_pool.browse(cr, uid, model_id, context=context).name
            template_name = "%s: %s" % (model_name, tools.ustr(record.subject))
            values = {
                'name': template_name,
                'subject': record.subject or False,
                'body_html': record.body or False,
                'model_id': model_id or False,
                'attachment_ids': [(6, 0, [att.id for att in record.attachment_ids])]
            }
            template_id = email_template.create(cr, uid, values, context=context)
            record.write(record.onchange_template_id(template_id, record.composition_mode, record.model, record.res_id)['value'])
            return _reopen(self, record.id, record.model)

    #------------------------------------------------------
    # Wizard validation and send
    #------------------------------------------------------

    def generate_email_for_composer(self, cr, uid, template_id, res_id, context=None):
        """ Call email_template.generate_email(), get fields relevant for
            mail.compose.message, transform email_cc and email_to into partner_ids """
        template_values = self.pool.get('email.template').generate_email(cr, uid, template_id, res_id, context=context)
        # filter template values
        fields = ['body_html', 'subject', 'email_to', 'email_recipients', 'email_cc', 'attachments']
        values = dict((field, template_values[field]) for field in fields if template_values.get(field))
        values['body'] = values.pop('body_html', '')

        # transform email_to, email_cc into partner_ids
        partner_ids = set()
        mails = tools.email_split(values.pop('email_to', '') + ' ' + values.pop('email_cc', ''))
        for mail in mails:
            partner_id = self.pool.get('res.partner').find_or_create(cr, uid, mail, context=context)
            partner_ids.add(partner_id)
        email_recipients = values.pop('email_recipients', '')
        if email_recipients:
            for partner_id in email_recipients.split(','):
                if partner_id:  # placeholders could generate '', 3, 2 due to some empty field values
                    partner_ids.add(int(partner_id))
        # legacy template behavior: void values do not erase existing values and the
        # related key is removed from the values dict
        if partner_ids:
            values['partner_ids'] = [(6, 0, list(partner_ids))]

        return values

    def render_message(self, cr, uid, wizard, res_id, context=None):
        """ Override to handle templates. """
        # generate the composer email
        if wizard.template_id:
            values = self.generate_email_for_composer(cr, uid, wizard.template_id, res_id, context=context)
        else:
            values = {}
        # get values to return
        email_dict = super(mail_compose_message, self).render_message(cr, uid, wizard, res_id, context)
        values.update(email_dict)
        return values

    def render_template(self, cr, uid, template, model, res_id, context=None):
        return self.pool.get('email.template').render_template(cr, uid, template, model, res_id, context=context)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>