<commit_before>from __future__ import unicode_literals
from six import StringIO

import re
import email
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header
from imbox.utils import str_encode, str_decode

class Struct(object):
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return str_decode(str_encode(value, default_charset, 'replace'), default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                headers[index] = str_decode(text, charset or default_charset,
                                            'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = str_decode(text, default_charset, 'replace')

        return ''.join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}

    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.search(r'=\?((?:\w|-)+)\?(Q|B)\?(.+)\?=', value)
        if match:
            encoding, type_, code = match.groups()
            if type_ == 'Q':
                value = quopri.decodestring(code)
            elif type_ == 'B':
                value = base64.decodestring(code)
            value = str_encode(value, encoding)
            value_results.append(value)
            if value_results:
                v = ''.join(value_results)
    return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None:
        dispositions = content_disposition.strip().split(";")

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)
            if file_data is None:
                return None

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': StringIO(file_data)
            }

            for param in dispositions[1:]:
                name, value = decode_param(param)

                if 'file' in name:
                    attachment['filename'] = value

                if 'create-date' in name:
                    attachment['create-date'] = value

            return attachment

    return None

def decode_content(message):
    try:
        content = message.get_payload(decode=True)
        charset = message.get_content_charset('utf-8')
        if charset != 'utf-8':
            return content.decode(charset)
        return content
    except:
        return u'Could not parse content'

def parse_email(raw_email):
    email_message = email.message_from_string(raw_email)
    maintype = email_message.get_content_maintype()
    parsed_email = {}

    parsed_email['raw_email'] = raw_email

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype == 'multipart':
        for part in email_message.walk():
            content_type = part.get_content_type()
            content_disposition = part.get('Content-Disposition', None)
            if content_disposition:
                content = part.get_payload(decode=True)
            else:
                content = decode_content(part)

            is_inline = content_disposition is None \
                or content_disposition == "inline"
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

    elif maintype == 'text':
        payload = decode_content(email_message)
        body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.items():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        timetuple = email.utils.parsedate(parsed_email['date'])
        parsed_date = datetime.fromtimestamp(time.mktime(timetuple)) \
            if timetuple else None
        parsed_email['parsed_date'] = parsed_date

    return Struct(**parsed_email)
<commit_msg>updated converter<commit_after>from __future__ import unicode_literals
from six import StringIO

import re
import email
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header
from imbox.utils import str_encode, str_decode

class Struct(object):
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return str_decode(str_encode(value, default_charset, 'replace'), default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                headers[index] = str_decode(text, charset or default_charset,
                                            'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = str_decode(text, default_charset, 'replace')

        return ''.join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}

    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.search(r'=\?((?:\w|-)+)\?(Q|B)\?(.+)\?=', value)
        if match:
            encoding, type_, code = match.groups()
            try:
                if isinstance(code, str):
                    code = code.encode('ascii')
            except Exception:
                pass
            if type_ == 'Q':
                value = quopri.decodestring(code)
            elif type_ == 'B':
                value = base64.decodebytes(code)
            value = str_encode(value, encoding)
            value_results.append(value)
            if value_results:
                v = ''.join(value_results)
    return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None:
        dispositions = content_disposition.strip().split(";")

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)
            if file_data is None:
                return None

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': StringIO(file_data)
            }

            for param in dispositions[1:]:
                name, value = decode_param(param)

                if 'file' in name:
                    attachment['filename'] = value

                if 'create-date' in name:
                    attachment['create-date'] = value

            return attachment

    return None

def decode_content(message):
    try:
        content = message.get_payload(decode=True)
        charset = message.get_content_charset('utf-8')
        if charset != 'utf-8':
            return content.decode(charset)
        return content
    except:
        return u'Could not parse content'

def parse_email(raw_email):
    email_message = email.message_from_string(raw_email)
    maintype = email_message.get_content_maintype()
    parsed_email = {}

    parsed_email['raw_email'] = raw_email

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype == 'multipart':
        for part in email_message.walk():
            content_type = part.get_content_type()
            content_disposition = part.get('Content-Disposition', None)
            if content_disposition:
                content = part.get_payload(decode=True)
            else:
                content = decode_content(part)

            is_inline = content_disposition is None \
                or content_disposition == "inline"
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

    elif maintype == 'text':
        payload = decode_content(email_message)
        body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.items():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        timetuple = email.utils.parsedate(parsed_email['date'])
        parsed_date = datetime.fromtimestamp(time.mktime(timetuple)) \
            if timetuple else None
        parsed_email['parsed_date'] = parsed_date

    return Struct(**parsed_email)
<|endoftext|>


<commit_before>#!/usr/bin/python
from buildutil import *

mgen_version = "NEEDS_TO_BE_SET"
mgen_cmd = "NEEDS_TO_BE_SET"
pluginPaths = "NEEDS_TO_BE_SET"
default_cpp_build_cfg = "NEEDS_TO_BE_SET"
java_projects = { # name: executable
    "api": False,
    "compiler": True,
    "javalib": False,
    "cppgenerator": False,
    "idlgenerator": False,
    "idlparser": False,
    "javagenerator": False,
    "javalib": False,
    "javascriptgenerator": False,
}

def compile4(workingDir, project, outPath, plug_paths):
    check_call(mgen_cmd + project + ' plugin_paths="' + plug_paths + '" output_path="' + outPath + '" use_env_vars="false"', cwd=workingDir, shell=True)

def compile3(workingDir, project, outPath):
    compile4(workingDir, project, outPath, pluginPaths)

def compile(workingDir, project):
    compile3(workingDir, project, ".")

def getCommitDateString():
    return os.popen("git show -s --format=%ci").read().rstrip()

def createJavaVersionFileContents(pkg, version):
    dateString = getCommitDateString()
    out = ""
    out += "package " + pkg + ";\n\n"
    out += "/**\n"
    out += " * Class generated to keep track of what MGen version this is\n"
    out += " */\n"
    out += "public class BuildVersion {\n"
    out += '   public static final String GIT_TAG = "' + version + '";\n'
    out += '   public static final String GIT_COMMIT_DATE = "' + dateString + '";\n'
    out += "}\n"
    return out

def createJavaVersionFile3(pkg, tgtFolder, version):
    fName = tgtFolder + "/BuildVersion.java"
    newFileContents = createJavaVersionFileContents(pkg, version)
    oldFileContents = file2String(fName)
    if (newFileContents != oldFileContents):
        mkFolder(tgtFolder)
        f = open(fName, "w")
        f.write(newFileContents)

def createJavaVersionFile2(project, version):
    pkg = "se.culvertsoft.mgen." + project
    tgtFolder = "mgen-" + project + "/src/main/java/se/culvertsoft/mgen/" + project
    createJavaVersionFile3(pkg, tgtFolder, version)

def createVersionFiles():
    for project in java_projects.keys():
        createJavaVersionFile2(project, mgen_version)

def build_jvm_parts():
    sbt('.', 'compile publish-local assembly')

def tests_generate_code(): # Ideally here we'd just generate once, not nLangs times.
    for lang in ["java", "cpp", "javascript"]:
        for model in ["project.xml", "transient_testmodel/project.xml", "defaultvalues_testmodel/project.xml", "defaultvaluesreq_testmodel/project.xml"]:
            compile4("mgen-" + lang + "lib", "../mgen-compiler/src/test/resources/" + model, ".", "../mgen-" + lang + "generator/target")
    for name in ["depends", "write", "read"]:
        compile3("mgen-integrationtests", 'models/'+name+'/project.xml', "generated/"+name)

def tests_integration_cpp():
    baseFolder = "mgen-integrationtests/generated"
    for name in ["depends", "write", "read"]:
        testFolder = baseFolder + "/" + name + "/data_generator"
        mkFolder(testFolder)
        mkFolder(baseFolder + "/" + name + "/data_generated")
        cmake(testFolder, "../../../build/" + name, default_cpp_build_cfg)
        cppBuildRun(testFolder, default_cpp_build_cfg, "generate_"+name+"_testdata")

def tests_integration_java():
    integFolder = "mgen-integrationtests"
    for name in ["depends", "write", "read"]:
        testFolder = integFolder + "/javacheck/" + name
        copyTree(integFolder + "/generated/" + name + "/src_generated/java", testFolder + "/src_generated/test/java")
        sbt_test(testFolder)

def tests_integration_js():
    integFolder = "mgen-integrationtests"
    for name in ["depends", "write", "read"]:
        testFolder = integFolder + "/javascriptcheck/" + name
        copyTree(integFolder + "/generated/" + name + "/src_generated/javascript", testFolder + "/src_generated/test/javascript")
        sbt_jasmine(testFolder)

def tests_normal():
    # Can this be done in one batch? Doesn't seem so,
    # seems like cwd is wierd when batching: test data isn't
    # copied over to the correct test dir - so we must make
    # sure to run tests from each wd separately
    sbt_test("mgen-javalib")
    sbt_test("mgen-javascriptlib")
    mkFolder("mgen-cpplib/target")
    cmake("mgen-cpplib/target", "../src/test/cpp/src", default_cpp_build_cfg)
    cppBuildRun("mgen-cpplib/target", default_cpp_build_cfg, "mgen-cpplib-test")

def copySubProjectJarFilesToZipDir(subprojectName, version, includeAssembly):
    srcFileBase = subprojectName + "/target/" + subprojectName + "-" + version
    destFileBase = "target/install_zip/jars/" + subprojectName
    copyFile(srcFileBase + ".jar", destFileBase + ".jar")
    copyFile(srcFileBase + "-sources.jar", destFileBase + "-sources.jar")
    copyFile(srcFileBase + "-javadoc.jar", destFileBase + "-javadoc.jar")
    if (includeAssembly):
        assemblySrc = subprojectName + "/target/" + subprojectName + "-assembly-" + version + ".jar"
        copyFile(assemblySrc, destFileBase + "-assembly.jar")

def copyJarFilesToZipDir():
    for project in java_projects.keys():
        copySubProjectJarFilesToZipDir("mgen-" + project, mgen_version, java_projects[project])

def getCppIncludeDir():
    return "mgen-cpplib/src/main/cpp"

def getInstallZipName():
    return "target/mgen-" + mgen_version + ".zip"

def create_install_zip():
    rmFolder("target/install_zip")
    mkFolder("target/install_zip")
    mkFolder("target/install_zip/jars")
    mkFolder("target/install_zip/bin")
    mkFolder("target/install_zip/javascript")

    copyJarFilesToZipDir()
    copyTree(getCppIncludeDir(), "target/install_zip/include")
    copyFile("mgen-javascriptlib/src/main/javascript/mgen-lib.js", "target/install_zip/javascript/mgen-lib.js")
    copyFile("mgen-starters/mgen.sh", "target/install_zip/bin/mgen")
    copyFile("mgen-starters/mgen.sh", "target/install_zip/bin/mgen.sh")
    copyFile("mgen-starters/mgen.ex_", "target/install_zip/bin/mgen.exe")
    copyFile("LICENSE", "target/install_zip/LICENSE.TXT")

    mkFileExecutable("target/install_zip/bin/mgen")
    mkFileExecutable("target/install_zip/bin/mgen.sh")

    versionFile = open("target/install_zip/BUILD.TXT", "w")
    versionFile.write("Release version: " + mgen_version + "\n")
    versionFile.write("Git commit date: " + getCommitDateString() + "\n")
    versionFile.close()

    zipdir("target/install_zip", getInstallZipName())

def install():

    #Check that we have an install path
    installPath = os.environ.get('MGEN_INSTALL_PATH')
    if installPath == None:
        raise Exception("Environmental variable MGEN_INSTALL_PATH not set")

    print("Installing MGEN to " + installPath)

    rmFolderContents(installPath)
    mkFolder(installPath)
    unzip(getInstallZipName(), installPath)
    mkFileExecutable(installPath + "/bin/mgen")
    mkFileExecutable(installPath + "/bin/mgen.sh")

def upload_to_culvertsoft():
    folderName = 'snapshot' if mgen_version == 'SNAPSHOT' else 'releases'
    zipName = "mgen-" + mgen_version + ".zip"
    zipSrc = "target/" + zipName
    zipDest = "www.culvertsoft.se:/var/www/" + folderName + "/" + zipName
    print("Uploading " +  zipName + " to " + zipDest)
    os.system("scp " + zipSrc + " " + zipDest)

def publish_to_sonatype():
    print("Publishing jar files to sonatype")
    sbt('.', 'publish-signed')
<commit_msg>more build script cleanup (still learning python here..)<commit_after>#!/usr/bin/python
from buildutil import *

mgen_version = "NEEDS_TO_BE_SET"
mgen_cmd = "NEEDS_TO_BE_SET"
pluginPaths = "NEEDS_TO_BE_SET"
default_cpp_build_cfg = "NEEDS_TO_BE_SET"
java_projects = { # name: executable
    "api": False,
    "compiler": True,
    "javalib": False,
    "cppgenerator": False,
    "idlgenerator": False,
    "idlparser": False,
    "javagenerator": False,
    "javalib": False,
    "javascriptgenerator": False,
}

def mgen_compile(workingDir, project, outPath, plug_paths = None):
    if not plug_paths: plug_paths = pluginPaths
    check_call(mgen_cmd + project + ' plugin_paths="' + plug_paths + '" output_path="' + outPath + '" use_env_vars="false"', cwd=workingDir, shell=True)

def getCommitDateString():
    return os.popen("git show -s --format=%ci").read().rstrip()

def createVersionFiles():
    def createJavaVersionFile2(project, version):
        def createJavaVersionFile3(pkg, tgtFolder):
            def createJavaVersionFileContents():
                dateString = getCommitDateString()
                out = ""
                out += "package " + pkg + ";\n\n"
                out += "/**\n"
                out += " * Class generated to keep track of what MGen version this is\n"
                out += " */\n"
                out += "public class BuildVersion {\n"
                out += '   public static final String GIT_TAG = "' + version + '";\n'
                out += '   public static final String GIT_COMMIT_DATE = "' + dateString + '";\n'
                out += "}\n"
                return out
            fName = tgtFolder + "/BuildVersion.java"
            newFileContents = createJavaVersionFileContents()
            oldFileContents = file2String(fName)
            if (newFileContents != oldFileContents):
                mkFolder(tgtFolder)
                f = open(fName, "w")
                f.write(newFileContents)
                f.close()
        pkg = "se.culvertsoft.mgen." + project
        tgtFolder = "mgen-" + project + "/src/main/java/se/culvertsoft/mgen/" + project
        createJavaVersionFile3(pkg, tgtFolder)
    for project in java_projects.keys():
        createJavaVersionFile2(project, mgen_version)

def build_jvm_parts():
    sbt('.', 'compile publish-local assembly')

def tests_generate_code(): # Ideally here we'd just generate once, not nLangs times.
    for lang in ["java", "cpp", "javascript"]:
        for model in ["project.xml", "transient_testmodel/project.xml", "defaultvalues_testmodel/project.xml", "defaultvaluesreq_testmodel/project.xml"]:
            mgen_compile("mgen-" + lang + "lib", "../mgen-compiler/src/test/resources/" + model, ".", "../mgen-" + lang + "generator/target")
    for name in ["depends", "write", "read"]:
        mgen_compile("mgen-integrationtests", 'models/'+name+'/project.xml', "generated/"+name)

def tests_integration_cpp():
    baseFolder = "mgen-integrationtests/generated"
    for name in ["depends", "write", "read"]:
        testFolder = baseFolder + "/" + name + "/data_generator"
        mkFolder(testFolder)
        mkFolder(baseFolder + "/" + name + "/data_generated")
        cmake(testFolder, "../../../build/" + name, default_cpp_build_cfg)
        cppBuildRun(testFolder, default_cpp_build_cfg, "generate_"+name+"_testdata")

def tests_integration_java():
    integFolder = "mgen-integrationtests"
    for name in ["depends", "write", "read"]:
        testFolder = integFolder + "/javacheck/" + name
        copyTree(integFolder + "/generated/" + name + "/src_generated/java", testFolder + "/src_generated/test/java")
        sbt_test(testFolder)

def tests_integration_js():
    integFolder = "mgen-integrationtests"
    for name in ["depends", "write", "read"]:
        testFolder = integFolder + "/javascriptcheck/" + name
        copyTree(integFolder + "/generated/" + name + "/src_generated/javascript", testFolder + "/src_generated/test/javascript")
        sbt_jasmine(testFolder)

def tests_normal():
    # Can this be done in one batch? Doesn't seem so,
    # seems like cwd is wierd when batching: test data isn't
    # copied over to the correct test dir - so we must make
    # sure to run tests from each wd separately
    sbt_test("mgen-javalib")
    sbt_test("mgen-javascriptlib")
    mkFolder("mgen-cpplib/target")
    cmake("mgen-cpplib/target", "../src/test/cpp/src", default_cpp_build_cfg)
    cppBuildRun("mgen-cpplib/target", default_cpp_build_cfg, "mgen-cpplib-test")

def getCppIncludeDir():
    return "mgen-cpplib/src/main/cpp"

def getInstallZipName():
    return "target/mgen-" + mgen_version + ".zip"

def create_install_zip():

    def copySubProjectJarFilesToZipDir(subprojectName, version, includeAssembly):
        srcFileBase = subprojectName + "/target/" + subprojectName + "-" + version
        destFileBase = "target/install_zip/jars/" + subprojectName
        copyFile(srcFileBase + ".jar", destFileBase + ".jar")
        copyFile(srcFileBase + "-sources.jar", destFileBase + "-sources.jar")
        copyFile(srcFileBase + "-javadoc.jar", destFileBase + "-javadoc.jar")
        if (includeAssembly):
            assemblySrc = subprojectName + "/target/" + subprojectName + "-assembly-" + version + ".jar"
            copyFile(assemblySrc, destFileBase + "-assembly.jar")

    rmFolder("target/install_zip")
    mkFolder("target/install_zip")
    mkFolder("target/install_zip/jars")
    mkFolder("target/install_zip/bin")
    mkFolder("target/install_zip/javascript")

    for project in java_projects.keys():
        copySubProjectJarFilesToZipDir("mgen-" + project, mgen_version, java_projects[project])

    copyTree(getCppIncludeDir(), "target/install_zip/include")
    copyFile("mgen-javascriptlib/src/main/javascript/mgen-lib.js", "target/install_zip/javascript/mgen-lib.js")
    copyFile("mgen-starters/mgen.sh", "target/install_zip/bin/mgen")
    copyFile("mgen-starters/mgen.sh", "target/install_zip/bin/mgen.sh")
    copyFile("mgen-starters/mgen.ex_", "target/install_zip/bin/mgen.exe")
    copyFile("LICENSE", "target/install_zip/LICENSE.TXT")
    versionFile = open("target/install_zip/BUILD.TXT", "w")
    versionFile.write("Release version: " + mgen_version + "\n")
    versionFile.write("Git commit date: " + getCommitDateString() + "\n")
    versionFile.close()

    zipdir("target/install_zip", getInstallZipName())

def install():

    #Check that we have an install path
    installPath = os.environ.get('MGEN_INSTALL_PATH')
    if installPath == None:
        raise Exception("Environmental variable MGEN_INSTALL_PATH not set")

    print("Installing MGEN to " + installPath)

    rmFolderContents(installPath)
    mkFolder(installPath)
    unzip(getInstallZipName(), installPath)
    mkFileExecutable(installPath + "/bin/mgen")
    mkFileExecutable(installPath + "/bin/mgen.sh")

def upload_to_culvertsoft():
    folderName = 'snapshot' if mgen_version == 'SNAPSHOT' else 'releases'
    zipName = "mgen-" + mgen_version + ".zip"
    zipSrc = "target/" + zipName
    zipDest = "www.culvertsoft.se:/var/www/" + folderName + "/" + zipName
    print("Uploading " +  zipName + " to " + zipDest)
    os.system("scp " + zipSrc + " " + zipDest)

def publish_to_sonatype():
    print("Publishing jar files to sonatype")
    sbt('.', 'publish-signed')
<|endoftext|>


<commit_before>from collections import namedtuple

Remote = namedtuple('Remote', ('name', 'url'))
CommitInfo = namedtuple("CommitInfo",
                        ('commit', 'origin', 'remote_repo', 'ref'))

class PRInfo(object):
    def __init__(self, json):
        self.json = json

    @property
    def base_sha(self):
        return self.json['base']['sha']

    @property
    def head_sha(self):
        return self.json['head']['sha']

    @property
    def base_ref(self):
        return self.json['base']['ref']

    @property
    def head_ref(self):
        return self.json['head']['ref']

    @property
    def has_remote_repo(self):
        return self.json['base']['repo']['owner']['login'] != \
               self.json['head']['repo']['owner']['login']

    @property
    def remote_repo(self):
        remote = None
        if self.has_remote_repo:
            remote = Remote(name=self.json['head']['repo']['owner']['login'],
                            url=self.json['head']['repo']['clone_url'])
        return remote

    def to_commit_info(self):
        return CommitInfo(self.base_sha, self.head_sha, self.remote_repo,
                          self.head_ref)

def get_pr_info(requester, reponame, number):
    "Returns the PullRequest as a PRInfo object"
    resp = requester.get(
        'https://api.github.com/repos/%s/pulls/%s' % (reponame, number))
    return PRInfo(resp.json())
<commit_msg>E127 continuation line over-indented for visual indent<commit_after>from collections import namedtuple

Remote = namedtuple('Remote', ('name', 'url'))
CommitInfo = namedtuple("CommitInfo",
                        ('commit', 'origin', 'remote_repo', 'ref'))

class PRInfo(object):
    def __init__(self, json):
        self.json = json

    @property
    def base_sha(self):
        return self.json['base']['sha']

    @property
    def head_sha(self):
        return self.json['head']['sha']

    @property
    def base_ref(self):
        return self.json['base']['ref']

    @property
    def head_ref(self):
        return self.json['head']['ref']

    @property
    def has_remote_repo(self):
        return self.json['base']['repo']['owner']['login'] != self.json['head']['repo']['owner']['login']

    @property
    def remote_repo(self):
        remote = None
        if self.has_remote_repo:
            remote = Remote(name=self.json['head']['repo']['owner']['login'],
                            url=self.json['head']['repo']['clone_url'])
        return remote

    def to_commit_info(self):
        return CommitInfo(self.base_sha, self.head_sha, self.remote_repo,
                          self.head_ref)

def get_pr_info(requester, reponame, number):
    "Returns the PullRequest as a PRInfo object"
    resp = requester.get(
        'https://api.github.com/repos/%s/pulls/%s' % (reponame, number))
    return PRInfo(resp.json())
<|endoftext|>


<commit_before>import io
import time
import picamera
from base_camera import BaseCamera

class Camera(BaseCamera):
    @staticmethod
    def frames():
        with picamera.PiCamera() as camera:
            # let camera warm up
            time.sleep(2)

            stream = io.BytesIO()
            for foo in camera.capture_continuous(stream, 'jpeg',
                                                 use_video_port=True):
                # return current frame
                stream.seek(0)
                yield stream.read()

                # reset stream for next frame
                stream.seek(0)
                stream.truncate()
<commit_msg>Update camera_pi.py<commit_after>import io
import time
import picamera
from base_camera import BaseCamera

class Camera(BaseCamera):
    @staticmethod
    def frames():
        with picamera.PiCamera() as camera:
            # let camera warm up
            time.sleep(2)

            stream = io.BytesIO()
            for _ in camera.capture_continuous(stream, 'jpeg',
                                                 use_video_port=True):
                # return current frame
                stream.seek(0)
                yield stream.read()

                # reset stream for next frame
                stream.seek(0)
                stream.truncate()
<|endoftext|>


<commit_before>import console
import editor
import keyword
import os
import string
import sys

ALLOWED_CHARS = set(string.ascii_letters + string.digits + string.whitespace + "_.")
DOCS = os.path.expanduser("~/Documents")

def preformat(name):
    name = str(name).strip()
    if name.startswith("from "):
        name = name[len("from "):].strip()
        if " import " in name:
            name = name[:name.rfind(" import ")].strip()
    elif name.startswith("import "):
        name = name[len("import "):].strip()

    if " as " in name:
        name = name[:name.rfind(" as ")].strip()

    return name

def is_valid_name(name):
    name = str(name).strip()

    # False strings are always invalid
    if not name:
        return False

    # Check for illegal chars by deleting all allowed ones and checking len
    if set(name) in ALLOWED_CHARS:
        return False

    # Check individual submodule parts
    parts = name.split(".")
    for part in parts:
        part = part.strip()

        # Skip part if empty
        if not part:
            continue

        # Check if part is keyword, starts with number or has invalid whitespace
        if keyword.iskeyword(part) or part[0] in string.digits or len(set(part) & set(string.whitespace)) > 0:
            return False

    # All tests passed, assumed a valid module name
    return True

def find_path(name):
    # Reject invalid module names
    if not is_valid_name(name):
        raise ValueError(name + " is not a valid module name")

    name = str(name).strip()

    # Check if built-in
    if name in sys.builtin_module_names:
        return "<built-in>"

    # Check all sys.path locations
    for p in sys.path:
        loc = os.path.abspath(os.path.join(p, name + ".py"))
        if os.path.exists(loc) and os.path.isfile(loc):
            return loc

    # Last resort, import module and check __file__
    try:
        mod = __import__(name)
    except ImportError:
        # Everything failed, assume nonexistant
        return "<string>"
    else:
        if hasattr(mod, "__file__"):
            return mod.__file__
        else:
            return "<built-in>"

def main():
    if len(sys.argv) > 1:
        name = sys.argv[1]
    else:
        sel = editor.get_selection() or ""
        seltxt = preformat(editor.get_text()[sel[0]:sel[1]])
        if sel and sel[0] != sel[1] and is_valid_name(seltxt):
            name = seltxt
        else:
            name = console.input_alert("Find Module")

    try:
        loc = find_path(name)
    except ValueError:
        console.hud_alert(name + " is not a valid module name", "error")
        return

    if loc == "<built-in>":
        console.hud_alert(name + " is a built-in module", "error")
    elif loc == "<string>":
        console.hud_alert("Could not find module " + name, "error")
    else:
        editor.open_file(os.path.relpath(loc, DOCS))

if __name__ == "__main__":
    main()
<commit_msg>Minor edits<commit_after>import console
import editor
import keyword
import os
import string
import sys

ALLOWED_CHARS = set(string.ascii_letters + string.digits + string.whitespace + "_.")
DOCS = os.path.expanduser("~/Documents")

def preformat(name):
    name = str(name).strip()
    if name.startswith("from "):
        name = name[len("from "):].strip()
        if " import " in name:
            name = name[:name.rfind(" import ")].strip()
    elif name.startswith("import "):
        name = name[len("import "):].strip()

    if " as " in name:
        name = name[:name.rfind(" as ")].strip()

    return name

def is_valid_name(name):
    name = str(name).strip()

    # False strings are always invalid
    if not name:
        return False

    # Check for illegal chars by deleting all allowed ones and checking len
    if set(name) in ALLOWED_CHARS:
        return False

    # Check individual submodule parts
    parts = name.split(".")
    for part in parts:
        part = part.strip()

        # Skip part if empty
        if not part:
            continue

        # Check if part is keyword, starts with number or has invalid whitespace
        if keyword.iskeyword(part) or part[0] in string.digits or len(set(part) & set(string.whitespace)) > 0:
            return False

    # All tests passed, assumed a valid module name
    return True

def find_path(name):
    # Reject invalid module names
    if not is_valid_name(name):
        raise ValueError(name + " is not a valid module name")

    name = str(name).strip()

    # Check if built-in
    if name in sys.builtin_module_names:
        return "<built-in>"

    # Check all sys.path locations
    for p in sys.path:
        loc = os.path.abspath(os.path.join(p, name + ".py"))
        if os.path.exists(loc) and os.path.isfile(loc):
            return loc

    # Last resort, import module and check __file__
    try:
        mod = __import__(name)
    except ImportError:
        # Everything failed, assume nonexistant
        return "<string>"
    return mod.__file__ if hasattr(mod, "__file__") else "<built-in>"

def main(args):
    if args:
        name = args[0]
    else:
        sel = editor.get_selection() or ""
        seltxt = preformat(editor.get_text()[sel[0]:sel[1]])
        if sel and sel[0] != sel[1] and is_valid_name(seltxt):
            name = seltxt
        else:
            name = console.input_alert("Find Module")

    try:
        loc = find_path(name)
    except ValueError:
        console.hud_alert(name + " is not a valid module name", "error")
        return

    if loc == "<built-in>":
        console.hud_alert(name + " is a built-in module", "error")
    elif loc == "<string>":
        console.hud_alert("Could not find module " + name, "error")
    else:
        editor.open_file(os.path.relpath(loc, DOCS))

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before># vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import cv2
import numpy as np
import utils
from numpy import matrix as mat

def generate_3d_points():
    ax=np.linspace(-2,2,8)
    xx,yy=np.meshgrid(ax,ax)
    return np.vstack((xx.flatten(),yy.flatten(),np.zeros(len(ax)**2))).T

def manuver1():
    #x,y,z,rx,ry,rz
    vec=np.zeros(6)
    vec[2]=3
    #vec[3]=-90/180.0*np.pi
    for _ in range(100):
        vec[2]+=0.1
        yield vec

    for _ in range(50):
        vec[3]+=1.0/180*np.pi
        yield vec

class Capture(object):
    def __init__(self,camera_matrix,size,noise_model=None):
        self.K=mat(camera_matrix).reshape(3,3)
        self.size=size
        self.manuever=manuver1()
        self.last_points=None
        self.last_position=None

    def read(self):
        self.last_position=self.manuever.__next__()
        x,y,z,rx,ry,rz=self.last_position
        img=np.zeros((self.size[0],self.size[1],3),dtype='uint8')
        C=np.array([x,y,z])
        #0=RC+T T=-R.T*C

        R=utils.eulerAnglesToRotationMatrix([rx,ry,rz])
        T=-mat(R).T*mat(C).T
        pts=(self.K*((mat(R)*generate_3d_points().T).T+T.T).T).T
        pts/=pts[:,2]
        for ptm in pts:
            pt=list(map(int,ptm.A1))
            #print(pt)
            cv2.rectangle(img,(pt[0]-1,pt[1]-1),(pt[0]+1,pt[1]+1),(0,255,0),1)
        #print(pts)
        self.last_points=pts[:,:2]
        return True,img

    def track(self,*args,**kargs):
        return self.last_points

if __name__=='__main__':
    cap=Capture([160.0,0,160, 0,160.0,120.0,0,0,1],(240,320))
    while 1:
        ret,im=cap.read()
        if ret:
            cv2.imshow('img',im)
            k=cv2.waitKey(0)%256
            if k==27:
                break
        else:
            import pdb;pdb.set_trace()

<commit_msg>add manuever<commit_after># vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import cv2
import numpy as np
import utils
from numpy import matrix as mat

def generate_3d_points():
    ax=np.linspace(-2,2,8)
    xx,yy=np.meshgrid(ax,ax)
    return np.vstack((xx.flatten(),yy.flatten(),np.zeros(len(ax)**2))).T

def manuver1():
    #x,y,z,rx,ry,rz
    vec=np.zeros(6)
    vec[2]=3
    #vec[3]=-90/180.0*np.pi
    for _ in range(100):
        vec[2]+=0.1
        yield vec

    for _ in range(50):
        vec[3]+=1.0/180*np.pi
        yield vec

def manuver2():
    #x,y,z,rx,ry,rz
    vec=np.zeros(6)
    vec[2]=3
    #vec[3]=-90/180.0*np.pi
    for _ in range(50):
        vec[2]+=0.1
        yield vec

    for rot_pitch,rot_roll in [(0,0),(200,20)]:
        for ind in range(400):
            if ind==0:
                step_x=-0.01
                step_y=0
            if ind==100:
                step_x=0
                step_y=0.01
            if ind==200:
                step_x=0.01
                step_y=0
            if ind==300:
                step_x=0
                step_y=-0.01
            vec[0]+=step_x
            vec[1]+=step_y

            if rot_pitch>0:
                pitch_sign=1 if (ind%rot_pitch)<(rot_pitch//2) else -1
                vec[3]+=pitch_sign*0.1/180.0*np.pi
            if rot_roll>0:
                roll_sign=1 if (ind%rot_roll)<(rot_roll//2) else -1
                vec[4]+=roll_sign*0.1/180.0*np.pi
            yield vec
    while 1:
            yield vec

class Capture(object):
    def __init__(self,camera_matrix,size,noise_model=None):
        self.K=mat(camera_matrix).reshape(3,3)
        self.size=size
        self.manuever=manuver2()
        self.last_points=None
        self.last_position=None

    def read(self):
        self.last_position=self.manuever.__next__()
        x,y,z,rx,ry,rz=self.last_position
        img=np.zeros((self.size[0],self.size[1],3),dtype='uint8')
        C=np.array([x,y,z])
        #0=RC+T T=-R.T*C

        R=utils.eulerAnglesToRotationMatrix([rx,ry,rz])
        T=-mat(R).T*mat(C).T
        pts=(self.K*((mat(R)*generate_3d_points().T).T+T.T).T).T
        pts/=pts[:,2]
        for ptm in pts:
            pt=list(map(int,ptm.A1))
            #print(pt)
            cv2.rectangle(img,(pt[0]-1,pt[1]-1),(pt[0]+1,pt[1]+1),(0,255,0),1)
        #print(pts)
        self.last_points=pts[:,:2]
        return True,img

    def track(self,*args,**kargs):
        return self.last_points

if __name__=='__main__':
    cap=Capture([160.0,0,160, 0,160.0,120.0,0,0,1],(240,320))
    while 1:
        ret,im=cap.read()
        if ret:
            cv2.imshow('img',im)
            k=cv2.waitKey(0)%256
            if k==27:
                break
        else:
            import pdb;pdb.set_trace()

<|endoftext|>


<commit_before><commit_msg>First version of small lib to use the importio platform<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import time
import unittest
import unittest.mock as mock
from datetime import datetime

import intelmq.lib.test as test
from intelmq.bots.outputs.elasticsearch.output import ElasticsearchOutputBot, get_event_date

if os.environ.get('INTELMQ_TEST_DATABASES'):
    import elasticsearch

INPUT1 = {"__type": "Event",
          "classification.type": "botnet drone",
          "source.asn": 64496,
          "source.ip": "192.0.2.1",
          "feed.name": "Example Feed",
          "extra": '{"foo.bar": "test"}'
          }
OUTPUT1 = {'classification.type': 'botnet drone',
           'extra.foo.bar': 'test',
           'feed.name': 'Example Feed',
           'source.asn': 64496,
           'source.ip': '192.0.2.1',
           }
ES_SEARCH = {"query": {
    "constant_score": {
        "filter": {
            "term": {
                "source.asn": 64496
            }
        }
    }
}
}
SAMPLE_TEMPLATE = {
    "mappings": {
        "events": {
            "properties": {
                "time.observation": {
                    "type": "date"
                },
                "time.source": {
                    "type": "date"
                },
                "classification.type": {
                    "type": "keyword"
                },
                "source.asn": {
                    "type": "integer"
                },
                "feed.name": {
                    "type": "text"
                },
                "source.ip": {
                    "type": "ip"
                }
            }
        }
    },
    "template": "intelmq-*",
    "index_patterns": [
        "intelmq-*"
    ]
}

TIMESTAMP_1 = "1869-12-02T00:00:00+00:00"
TIMESTAMP_2 = "2020-02-02T01:23:45+00:00"
TIMESTAMP_3 = "2018-09-09T01:23:45+00:00"
INPUT_TIME_SOURCE = {
    "__type": "Event",
    "classification.type": "botnet drone",
    "source.asn": 64496,
    "source.ip": "192.0.2.1",
    "feed.name": "Example Feed",
    "time.source": TIMESTAMP_1,
    "extra": '{"foo.bar": "test"}'
}
INPUT_TIME_OBSERVATION = {
    "__type": "Event",
    "classification.type": "botnet drone",
    "source.asn": 64496,
    "source.ip": "192.0.2.1",
    "feed.name": "Example Feed",
    "time.observation": TIMESTAMP_2,
    "extra": '{"foo.bar": "test"}'
}

ROTATE_OPTIONS = {
    'never': None,
    'daily': '%Y-%m-%d',
    'weekly': '%Y-%W',
    'monthly': '%Y-%m',
    'yearly': '%Y'
}

@test.skip_database()
class TestElasticsearchOutputBot(test.BotTestCase, unittest.TestCase):

    @classmethod
    def set_bot(cls):
        cls.bot_reference = ElasticsearchOutputBot
        cls.default_input_message = INPUT1
        cls.sysconfig = {"flatten_fields": "extra",
                         "rotate_index": "never"}
        if os.environ.get('INTELMQ_TEST_DATABASES'):
            cls.con = elasticsearch.Elasticsearch()

    def test_event(self):
        self.run_bot()
        time.sleep(1)  # ES needs some time between inserting and searching
        result = self.con.search(index='intelmq', body=ES_SEARCH)['hits']['hits'][0]
        self.con.delete(index='intelmq', doc_type='events', id=result['_id'])
        self.assertDictEqual(OUTPUT1, result['_source'])

    def test_raise_when_no_template(self):
        """
        Test that a bot raises a RuntimeError if 'rotate_index' is set, but a matching template doesn't exist in ES.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        self.assertRaises(RuntimeError, self.run_bot())

    def test_get_event_date(self):
        """
        Test whether get_event_date detects the time.source and time.observation fields in an event.
        """
        self.assertEqual(get_event_date(INPUT_TIME_SOURCE),
                         datetime.strptime(TIMESTAMP_1, '%Y-%m-%dT%H:%M:%S+00:00').date())
        self.assertEqual(get_event_date(INPUT_TIME_OBSERVATION),
                         datetime.strptime(TIMESTAMP_2, '%Y-%m-%dT%H:%M:%S+00:00').date())

    def test_index_detected_from_time_source(self):
        """
        Tests whether an input event with a time.source field is indexed according to its time.source date.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        expected_index_name = "{}-1869-12-02".format(self.sysconfig.get('elastic_index'))
        self.base_check_expected_index_created(INPUT_TIME_SOURCE, expected_index_name)

    def test_index_detected_from_time_observation(self):
        """
        Tests whether an input event with a time.observation field (and no time.source field) is indexed according to
        its time.observation date.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        expected_index_name = "{}-2020-02-02".format(self.sysconfig.get('elastic_index'))
        self.base_check_expected_index_created(INPUT_TIME_OBSERVATION, expected_index_name)

    def test_index_falls_back_to_default_date(self):
        """
        Tests whether get_index returns an expected default date
         if no time.source or time.observation is present, and
         that the OutputBot will use the current date if none is present.
        """

        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}

        class FakeDateTime(datetime):
            """
            Passed to bot to force expected datetime value for test.
            """
            @classmethod
            def today(cls):
                return datetime.strptime('2018-09-09T01:23:45+00:00', '%Y-%m-%dT%H:%M:%S+00:00')

        expected_index_name = "{}-{}".format(self.sysconfig.get('elastic_index'), "2018-09-09")

        # Patch datetime with FakeDateTime, run the bot, and check the created index.
        with mock.patch('intelmq.bots.outputs.elasticsearch.output.datetime', new=FakeDateTime):
            self.base_check_expected_index_created(INPUT1, expected_index_name)

    def test_index_falls_back_to_default_string(self):
        """
        Tests whether get_index returns an expected default string
         if no time.source or time.observation is present, and
         that the OutputBot will use the current date if none is present.
        """

        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}

        self.prepare_bot()
        index = self.bot.get_index(INPUT1, default_string='test-default')
        self.assertEqual(index, 'intelmq-test-default')  # Check that get_index honors the supplied default string

    def base_check_expected_index_created(self, input_event, expected_index_name):
        self.input_message = input_event

        self.assertFalse(self.con.indices.exists(expected_index_name))  # Index should not already exist

        # Create the template mapping in Elasticsearch
        self.con.indices.put_template(name=self.sysconfig.get('elastic_index'), body=SAMPLE_TEMPLATE)

        self.run_bot()
        time.sleep(1)  # Let ES store the event. Can also force this with ES API

        self.assertTrue(self.con.indices.exists(expected_index_name))

        result = self.con.search(index=expected_index_name, body=ES_SEARCH)['hits']['hits'][0]
        result_index_name = result["_index"]

        # Clean up test event and check that the index name was set correctly
        self.con.delete(index=result_index_name, doc_type=self.sysconfig.get('elastic_doctype'), id=result['_id'])
        self.assertEqual(result_index_name, expected_index_name)

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<commit_msg>Run bot and clear output queue in default index string test<commit_after># -*- coding: utf-8 -*-
import os
import time
import unittest
import unittest.mock as mock
from datetime import datetime

import intelmq.lib.test as test
from intelmq.bots.outputs.elasticsearch.output import ElasticsearchOutputBot, get_event_date

if os.environ.get('INTELMQ_TEST_DATABASES'):
    import elasticsearch

INPUT1 = {"__type": "Event",
          "classification.type": "botnet drone",
          "source.asn": 64496,
          "source.ip": "192.0.2.1",
          "feed.name": "Example Feed",
          "extra": '{"foo.bar": "test"}'
          }
OUTPUT1 = {'classification.type': 'botnet drone',
           'extra.foo.bar': 'test',
           'feed.name': 'Example Feed',
           'source.asn': 64496,
           'source.ip': '192.0.2.1',
           }
ES_SEARCH = {"query": {
    "constant_score": {
        "filter": {
            "term": {
                "source.asn": 64496
            }
        }
    }
}
}
SAMPLE_TEMPLATE = {
    "mappings": {
        "events": {
            "properties": {
                "time.observation": {
                    "type": "date"
                },
                "time.source": {
                    "type": "date"
                },
                "classification.type": {
                    "type": "keyword"
                },
                "source.asn": {
                    "type": "integer"
                },
                "feed.name": {
                    "type": "text"
                },
                "source.ip": {
                    "type": "ip"
                }
            }
        }
    },
    "template": "intelmq-*",
    "index_patterns": [
        "intelmq-*"
    ]
}

TIMESTAMP_1 = "1869-12-02T00:00:00+00:00"
TIMESTAMP_2 = "2020-02-02T01:23:45+00:00"
TIMESTAMP_3 = "2018-09-09T01:23:45+00:00"
INPUT_TIME_SOURCE = {
    "__type": "Event",
    "classification.type": "botnet drone",
    "source.asn": 64496,
    "source.ip": "192.0.2.1",
    "feed.name": "Example Feed",
    "time.source": TIMESTAMP_1,
    "extra": '{"foo.bar": "test"}'
}
INPUT_TIME_OBSERVATION = {
    "__type": "Event",
    "classification.type": "botnet drone",
    "source.asn": 64496,
    "source.ip": "192.0.2.1",
    "feed.name": "Example Feed",
    "time.observation": TIMESTAMP_2,
    "extra": '{"foo.bar": "test"}'
}

ROTATE_OPTIONS = {
    'never': None,
    'daily': '%Y-%m-%d',
    'weekly': '%Y-%W',
    'monthly': '%Y-%m',
    'yearly': '%Y'
}

@test.skip_database()
class TestElasticsearchOutputBot(test.BotTestCase, unittest.TestCase):

    @classmethod
    def set_bot(cls):
        cls.bot_reference = ElasticsearchOutputBot
        cls.default_input_message = INPUT1
        cls.sysconfig = {"flatten_fields": "extra",
                         "rotate_index": "never"}
        if os.environ.get('INTELMQ_TEST_DATABASES'):
            cls.con = elasticsearch.Elasticsearch()

    def test_event(self):
        self.run_bot()
        time.sleep(1)  # ES needs some time between inserting and searching
        result = self.con.search(index='intelmq', body=ES_SEARCH)['hits']['hits'][0]
        self.con.delete(index='intelmq', doc_type='events', id=result['_id'])
        self.assertDictEqual(OUTPUT1, result['_source'])

    def test_raise_when_no_template(self):
        """
        Test that a bot raises a RuntimeError if 'rotate_index' is set, but a matching template doesn't exist in ES.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        self.assertRaises(RuntimeError, self.run_bot())

    def test_get_event_date(self):
        """
        Test whether get_event_date detects the time.source and time.observation fields in an event.
        """
        self.assertEqual(get_event_date(INPUT_TIME_SOURCE),
                         datetime.strptime(TIMESTAMP_1, '%Y-%m-%dT%H:%M:%S+00:00').date())
        self.assertEqual(get_event_date(INPUT_TIME_OBSERVATION),
                         datetime.strptime(TIMESTAMP_2, '%Y-%m-%dT%H:%M:%S+00:00').date())

    def test_index_detected_from_time_source(self):
        """
        Tests whether an input event with a time.source field is indexed according to its time.source date.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        expected_index_name = "{}-1869-12-02".format(self.sysconfig.get('elastic_index'))
        self.base_check_expected_index_created(INPUT_TIME_SOURCE, expected_index_name)

    def test_index_detected_from_time_observation(self):
        """
        Tests whether an input event with a time.observation field (and no time.source field) is indexed according to
        its time.observation date.
        """
        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}
        expected_index_name = "{}-2020-02-02".format(self.sysconfig.get('elastic_index'))
        self.base_check_expected_index_created(INPUT_TIME_OBSERVATION, expected_index_name)

    def test_index_falls_back_to_default_date(self):
        """
        Tests whether get_index returns an expected default date
         if no time.source or time.observation is present, and
         that the OutputBot will use the current date if none is present.
        """

        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}

        class FakeDateTime(datetime):
            """
            Passed to bot to force expected datetime value for test.
            """
            @classmethod
            def today(cls):
                return datetime.strptime('2018-09-09T01:23:45+00:00', '%Y-%m-%dT%H:%M:%S+00:00')

        expected_index_name = "{}-{}".format(self.sysconfig.get('elastic_index'), "2018-09-09")

        # Patch datetime with FakeDateTime, run the bot, and check the created index.
        with mock.patch('intelmq.bots.outputs.elasticsearch.output.datetime', new=FakeDateTime):
            self.base_check_expected_index_created(INPUT1, expected_index_name)

    def test_index_falls_back_to_default_string(self):
        """
        Tests whether get_index returns an expected default string
         if no time.source or time.observation is present, and
         that the OutputBot will use the current date if none is present.
        """

        self.sysconfig = {"flatten_fields": "extra",
                          "elastic_index": "intelmq",
                          "elastic_doctype": "events",
                          "rotate_index": "daily"}

        self.prepare_bot()
        index = self.bot.get_index(INPUT1, default_string='test-default')
        self.assertEqual(index, 'intelmq-test-default')  # Check that get_index honors the supplied default string
        self.run_bot()  # Run to clear output queue. Will still index according to current date.

    def base_check_expected_index_created(self, input_event, expected_index_name):
        self.input_message = input_event

        self.assertFalse(self.con.indices.exists(expected_index_name))  # Index should not already exist

        # Create the template mapping in Elasticsearch
        self.con.indices.put_template(name=self.sysconfig.get('elastic_index'), body=SAMPLE_TEMPLATE)

        self.run_bot()
        time.sleep(1)  # Let ES store the event. Can also force this with ES API

        self.assertTrue(self.con.indices.exists(expected_index_name))

        result = self.con.search(index=expected_index_name, body=ES_SEARCH)['hits']['hits'][0]
        result_index_name = result["_index"]

        # Clean up test event and check that the index name was set correctly
        self.con.delete(index=result_index_name, doc_type=self.sysconfig.get('elastic_doctype'), id=result['_id'])
        self.assertEqual(result_index_name, expected_index_name)

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<|endoftext|>


<commit_before>import requests
from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError, Timeout, RequestException
from urllib3.util.retry import Retry

def retryable_session(retries=3, backoff_factor=0.5, status_forcelist=(500, 502, 504, 520), session=None):
    # from https://www.peterbe.com/plog/best-practice-with-retries-with-requests
    session = session or requests.Session()
    # 'Retry-After' 413/503/529 headers are respected by default
    retry = Retry(total=retries, read=retries, connect=retries,
        backoff_factor=backoff_factor, status_forcelist=status_forcelist)
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

class Getter(object):
    def __init__(self, contenttype=None, login=lambda: False, session=None):
        self.session = session or retryable_session()
        self.login = login
        if contenttype:
            self.session.headers['Accept'] = contenttype

    def __call__(self, url, **kwargs):
        try:
            return self._inner_call(url, **kwargs)
        except (Timeout, ConnectionError, RequestException) as ex:
            message = ex.response.reason if getattr(ex, 'response', None) else type(ex).__name__
            raise GetterError(message, ex, not isinstance(ex, RequestException))

    def _inner_call(self, url, **kwargs):
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 20
        result = self.session.get(url, **kwargs)
        if result is None:
            return
        if result.status_code == 401:
            if self.login():
                result = self.session.get(url, **kwargs)
                if result is None:
                    return

        if result.status_code == 404:
            return
        result.raise_for_status()
        return result

class GetterError(Exception):
    def __init__(self, message, cause, connection_error):
        super(GetterError, self).__init__()
        self.message = message
        self.cause = cause
        self.connection_error = connection_error
        self.request = getattr(cause, 'request', None)
        self.response = getattr(cause, 'response', None)
<commit_msg>fix: import urllib3 from requests<commit_after>import requests
from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError, Timeout, RequestException
# import from `requests` because Jarvis / some platforms still have old urllib3
from requests.packages.urllib3.util.retry import Retry

def retryable_session(retries=3, backoff_factor=0.5, status_forcelist=(500, 502, 504, 520), session=None):
    # from https://www.peterbe.com/plog/best-practice-with-retries-with-requests
    session = session or requests.Session()
    # 'Retry-After' 413/503/529 headers are respected by default
    retry = Retry(total=retries, read=retries, connect=retries,
        backoff_factor=backoff_factor, status_forcelist=status_forcelist)
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

class Getter(object):
    def __init__(self, contenttype=None, login=lambda: False, session=None):
        self.session = session or retryable_session()
        self.login = login
        if contenttype:
            self.session.headers['Accept'] = contenttype

    def __call__(self, url, **kwargs):
        try:
            return self._inner_call(url, **kwargs)
        except (Timeout, ConnectionError, RequestException) as ex:
            message = ex.response.reason if getattr(ex, 'response', None) else type(ex).__name__
            raise GetterError(message, ex, not isinstance(ex, RequestException))

    def _inner_call(self, url, **kwargs):
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 20
        result = self.session.get(url, **kwargs)
        if result is None:
            return
        if result.status_code == 401:
            if self.login():
                result = self.session.get(url, **kwargs)
                if result is None:
                    return

        if result.status_code == 404:
            return
        result.raise_for_status()
        return result

class GetterError(Exception):
    def __init__(self, message, cause, connection_error):
        super(GetterError, self).__init__()
        self.message = message
        self.cause = cause
        self.connection_error = connection_error
        self.request = getattr(cause, 'request', None)
        self.response = getattr(cause, 'response', None)
<|endoftext|>


<commit_before>"""Configuration loading and objects."""

import pathlib
import threading
import configparser
import collections.abc

from jacquard.storage import open_engine
from jacquard.directory import open_directory

class Config(collections.abc.Mapping):
    """
    System configuration.

    Users should not instantiate this class directly but instead use the
    public `load_config` function.
    """

    def __init__(self, config_file):
        """Internal constructor."""
        self.storage_engine = config_file.get('storage', 'engine')
        self.storage_url = config_file.get('storage', 'url', fallback='')
        self.directory_settings = config_file['directory']
        self.config_file = config_file

        self._thread_local = threading.local()

    def __getitem__(self, key):
        """Look up config section by name."""
        return self.config_file[key]

    def __len__(self):
        """Total number of config sections."""
        return len(self.config_file)

    def __iter__(self):
        """Iterator over names of config sections."""
        return iter(self.config_file)

    def __contains__(self, key):
        """Existence of config section."""
        return key in self.config_file

    def _thread_local_property(self, name, generator):
        if not hasattr(self._thread_local, name):
            setattr(self._thread_local, name, generator())
        return getattr(self._thread_local, name)

    def _open_storage(self):
        return open_engine(self, self.storage_engine, self.storage_url)

    @property
    def storage(self):
        """
        Key-value storage engine.

        Note that this is actually thread-local, so users need not worry
        about thread synchronisation of connections.
        """
        return self._thread_local_property('storage', self._open_storage)

    def _open_directory(self):
        kwargs = {
            key: value
            for key, value in self.directory_settings.items()
            if key != 'engine'
        }
        return open_directory(
            self,
            self.directory_settings['engine'],
            kwargs,
        )

    @property
    def directory(self):
        """
        User lookup directory.

        Note that this is actually thread-local, so users need not worry
        about thread synchronisation of connections.
        """
        return self._thread_local_property('directory', self._open_directory)

def load_config(source):
    """
    Load `Config` from file.

    `source` may be given either as a file-like or path-like object. In the
    former case it should be opened for reading in text/unicode mode.
    """
    if hasattr(source, 'read'):
        return _load_config_from_fp(source)
    else:
        with pathlib.Path(source).open('r') as f:
            return _load_config_from_fp(f)

def _load_config_from_fp(f):
    parser = configparser.ConfigParser()
    parser.read_file(f)

    return Config(parser)
<commit_msg>Add custom paths to config<commit_after>"""Configuration loading and objects."""

import sys
import pathlib
import threading
import configparser
import collections.abc

from jacquard.storage import open_engine
from jacquard.directory import open_directory

class Config(collections.abc.Mapping):
    """
    System configuration.

    Users should not instantiate this class directly but instead use the
    public `load_config` function.
    """

    def __init__(self, config_file):
        """Internal constructor."""
        self.storage_engine = config_file.get('storage', 'engine')
        self.storage_url = config_file.get('storage', 'url', fallback='')
        self.directory_settings = config_file['directory']
        self.config_file = config_file

        self._load_path()

        self._thread_local = threading.local()

    def _load_path(self):
        for path in self.get('paths', {}).values():
            sys.path.append(path.strip())

    def __getitem__(self, key):
        """Look up config section by name."""
        return self.config_file[key]

    def __len__(self):
        """Total number of config sections."""
        return len(self.config_file)

    def __iter__(self):
        """Iterator over names of config sections."""
        return iter(self.config_file)

    def __contains__(self, key):
        """Existence of config section."""
        return key in self.config_file

    def _thread_local_property(self, name, generator):
        if not hasattr(self._thread_local, name):
            setattr(self._thread_local, name, generator())
        return getattr(self._thread_local, name)

    def _open_storage(self):
        return open_engine(self, self.storage_engine, self.storage_url)

    @property
    def storage(self):
        """
        Key-value storage engine.

        Note that this is actually thread-local, so users need not worry
        about thread synchronisation of connections.
        """
        return self._thread_local_property('storage', self._open_storage)

    def _open_directory(self):
        kwargs = {
            key: value
            for key, value in self.directory_settings.items()
            if key != 'engine'
        }
        return open_directory(
            self,
            self.directory_settings['engine'],
            kwargs,
        )

    @property
    def directory(self):
        """
        User lookup directory.

        Note that this is actually thread-local, so users need not worry
        about thread synchronisation of connections.
        """
        return self._thread_local_property('directory', self._open_directory)

def load_config(source):
    """
    Load `Config` from file.

    `source` may be given either as a file-like or path-like object. In the
    former case it should be opened for reading in text/unicode mode.
    """
    if hasattr(source, 'read'):
        return _load_config_from_fp(source)
    else:
        with pathlib.Path(source).open('r') as f:
            return _load_config_from_fp(f)

def _load_config_from_fp(f):
    parser = configparser.ConfigParser()
    parser.read_file(f)

    return Config(parser)
<|endoftext|>


<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#

import os, copy, zlib

from jasy.core.Error import JasyError

import jasy.js.parse.Parser as Parser
import jasy.js.parse.ScopeScanner as ScopeScanner

import jasy.js.clean.DeadCode
import jasy.js.clean.Unused
import jasy.js.clean.Permutate

import jasy.js.optimize.Translation

import jasy.js.output.Optimization

from jasy.item.Item import Item
from jasy.core.Permutation import getPermutation
from jasy.js.api.Data import ApiData
from jasy.js.MetaData import MetaData
from jasy.js.output.Compressor import Compressor

from jasy.js.util import *

from jasy.js.optimize.Translation import hasText, collectTranslations

from jasy.core.Logging import *

try:
    from pygments import highlight
    from pygments.lexers import JavascriptLexer
    from pygments.formatters import HtmlFormatter
except:
    highlight = None

aliases = {}

defaultOptimization = jasy.js.output.Optimization.Optimization("declarations", "blocks", "variables")
defaultPermutation = getPermutation({"debug" : False})

__all__ = ["Class", "Error"]

def collectFields(node, keys=None):

    if keys is None:
        keys = set()

    # Always the first parameter
    # Supported calls: core.Env.isSet(key, expected?), core.Env.getValue(key), core.Env.select(key, map)
    calls = ("core.Env.isSet", "core.Env.getValue", "core.Env.select")
    if node.type == "dot" and node.parent.type == "call" and assembleDot(node) in calls:
        keys.add(node.parent[1][0].value)

    # Process children
    for child in reversed(node):
        if child != None:
            collectFields(child, keys)

    return keys

class ClassError(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing class %s: %s" % (self.__inst, self.__msg)

class Class(Item):

    kind = "class"

    def __getTree(self, context=None):

        field = "tree[%s]" % self.id
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            info("Processing class %s %s...", colorize(self.id, "bold"), colorize("[%s]" % context, "cyan"))

            indent()
            tree = Parser.parse(self.getText(), self.id)
            ScopeScanner.scan(tree)
            outdent()

            self.project.getCache().store(field, tree, self.mtime, True)

        return tree

    def __getOptimizedTree(self, permutation=None, context=None):
        """Returns an optimized tree with permutations applied"""

        field = "opt-tree[%s]-%s" % (self.id, permutation)
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            tree = copy.deepcopy(self.__getTree("%s:plain" % context))

            # Logging
            msg = "Processing class %s" % colorize(self.id, "bold")
            if permutation:
                msg += colorize(" (%s)" % permutation, "grey")
            if context:
                msg += colorize(" [%s]" % context, "cyan")

            info("%s..." % msg)
            indent()

            # Apply permutation
            if permutation:
                jasy.js.clean.Permutate.patch(tree, permutation)

            # Cleanups
            jasy.js.clean.DeadCode.cleanup(tree)
            ScopeScanner.scan(tree)
            jasy.js.clean.Unused.cleanup(tree)

            self.project.getCache().store(field, tree, self.mtime, True)
            outdent()

        return tree

    def getDependencies(self, permutation=None, classes=None, warnings=True):
        """
        Returns a set of dependencies seen through the given list of known
        classes (ignoring all unknown items in original set). This method
        makes use of the meta data (see core/MetaData.py) and the variable data
        (see parse/ScopeData.py).
        """

        permutation = self.filterPermutation(permutation)

        meta = self.getMetaData(permutation)
        scope = self.getScopeData(permutation)

        result = set()

        # Manually defined names/classes
        for name in meta.requires:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.add(classes[name])
            elif warnings:
                warn("- Missing class (required): %s in %s", name, self.id)

        # Globally modified names (mostly relevant when working without namespaces)
        for name in scope.shared:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.add(classes[name])

        # Add classes from detected package access
        for package in scope.packages:
            if package in aliases:
                className = aliases[package]
                if className in classes:
                    result.add(classes[className])
                    continue

            orig = package
            while True:
                if package == self.id:
                    break

                elif package in classes and classes[package].kind == "class":
                    aliases[orig] = package
                    result.add(classes[package])
                    break

                else:
                    pos = package.rfind(".")
                    if pos == -1:
                        break

                    package = package[0:pos]

        # Manually excluded names/classes
        for name in meta.optionals:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.remove(classes[name])
            elif warnings:
                warn("- Missing class (optional): %s in %s", name, self.id)

        return result

    def getScopeData(self, permutation=None):
        """
        Returns the top level scope object which contains information about the
        global variable and package usage/influence.
        """

        permutation = self.filterPermutation(permutation)

        field = "scope[%s]-%s" % (self.id, permutation)
        scope = self.project.getCache().read(field, self.mtime)
        if scope is None:
            scope = self.__getOptimizedTree(permutation, "scope").scope
            self.project.getCache().store(field, scope, self.mtime)

        return scope

    def getApi(self, highlight=True):
        field = "api[%s]-%s" % (self.id, highlight)
        apidata = self.project.getCache().read(field, self.mtime, inMemory=False)
        if apidata is None:
            apidata = ApiData(self.id, highlight)

            tree = self.__getTree(context="api")
            indent()
            apidata.scanTree(tree)
            outdent()

            metaData = self.getMetaData()
            apidata.addAssets(metaData.assets)
            for require in metaData.requires:
                apidata.addUses(require)
            for optional in metaData.optionals:
                apidata.removeUses(optional)

            apidata.addSize(self.getSize())
            apidata.addFields(self.getFields())

            self.project.getCache().store(field, apidata, self.mtime, inMemory=False)

        return apidata

    def getHighlightedCode(self):
        field = "highlighted[%s]" % self.id
        source = self.project.getCache().read(field, self.mtime)
        if source is None:
            if highlight is None:
                raise JasyError("Could not highlight JavaScript code! Please install Pygments.")

            lexer = JavascriptLexer(tabsize=2)
            formatter = HtmlFormatter(full=True, style="autumn", linenos="table", lineanchors="line")
            source = highlight(self.getText(), lexer, formatter)

            self.project.getCache().store(field, source, self.mtime)

        return source

    def getMetaData(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "meta[%s]-%s" % (self.id, permutation)
        meta = self.project.getCache().read(field, self.mtime)
        if meta is None:
            meta = MetaData(self.__getOptimizedTree(permutation, "meta"))
            self.project.getCache().store(field, meta, self.mtime)

        return meta

    def getFields(self):
        field = "fields[%s]" % (self.id)
        fields = self.project.getCache().read(field, self.mtime)
        if fields is None:
            fields = collectFields(self.__getTree(context="fields"))
            self.project.getCache().store(field, fields, self.mtime)

        return fields

    def getTranslations(self):
        field = "translations[%s]" % (self.id)
        result = self.project.getCache().read(field, self.mtime)
        if result is None:
            result = collectTranslations(self.__getTree(context="i18n"))
            self.project.getCache().store(field, result, self.mtime)

        return result

    def filterPermutation(self, permutation):
        if permutation:
            fields = self.getFields()
            if fields:
                return permutation.filter(fields)

        return None

    def getCompressed(self, permutation=None, translation=None, optimization=None, formatting=None, context="compressed"):
        permutation = self.filterPermutation(permutation)

        # Disable translation for caching / patching when not actually used
        if translation and not self.getTranslations():
            translation = None

        field = "compressed[%s]-%s-%s-%s-%s" % (self.id, permutation, translation, optimization, formatting)
        compressed = self.project.getCache().read(field, self.mtime)
        if compressed == None:
            tree = self.__getOptimizedTree(permutation, context)

            if translation or optimization:
                tree = copy.deepcopy(tree)

                if translation:
                    jasy.js.optimize.Translation.optimize(tree, translation)

                if optimization:
                    try:
                        optimization.apply(tree)
                    except jasy.js.output.Optimization.Error as error:
                        raise ClassError(self, "Could not compress class! %s" % error)

            compressed = Compressor(formatting).compress(tree)
            self.project.getCache().store(field, compressed, self.mtime)

        return compressed

    def getSize(self):
        field = "size[%s]" % self.id
        size = self.project.getCache().read(field, self.mtime)

        if size is None:
            compressed = self.getCompressed(context="size")
            optimized = self.getCompressed(permutation=defaultPermutation, optimization=defaultOptimization, context="size")
            zipped = zlib.compress(optimized.encode("utf-8"))

            size = {
                "compressed" : len(compressed),
                "optimized" : len(optimized),
                "zipped" : len(zipped)
            }

            self.project.getCache().store(field, size, self.mtime)

        return size

<commit_msg>Minor optimization<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#

import os, copy, zlib

from jasy.core.Error import JasyError

import jasy.js.parse.Parser as Parser
import jasy.js.parse.ScopeScanner as ScopeScanner

import jasy.js.clean.DeadCode
import jasy.js.clean.Unused
import jasy.js.clean.Permutate

import jasy.js.optimize.Translation

import jasy.js.output.Optimization

from jasy.item.Item import Item
from jasy.core.Permutation import getPermutation
from jasy.js.api.Data import ApiData
from jasy.js.MetaData import MetaData
from jasy.js.output.Compressor import Compressor

from jasy.js.util import *

from jasy.core.Logging import *

try:
    from pygments import highlight
    from pygments.lexers import JavascriptLexer
    from pygments.formatters import HtmlFormatter
except:
    highlight = None

aliases = {}

defaultOptimization = jasy.js.output.Optimization.Optimization("declarations", "blocks", "variables")
defaultPermutation = getPermutation({"debug" : False})

__all__ = ["Class", "Error"]

def collectFields(node, keys=None):

    if keys is None:
        keys = set()

    # Always the first parameter
    # Supported calls: core.Env.isSet(key, expected?), core.Env.getValue(key), core.Env.select(key, map)
    calls = ("core.Env.isSet", "core.Env.getValue", "core.Env.select")
    if node.type == "dot" and node.parent.type == "call" and assembleDot(node) in calls:
        keys.add(node.parent[1][0].value)

    # Process children
    for child in reversed(node):
        if child != None:
            collectFields(child, keys)

    return keys

class ClassError(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing class %s: %s" % (self.__inst, self.__msg)

class Class(Item):

    kind = "class"

    def __getTree(self, context=None):

        field = "tree[%s]" % self.id
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            info("Processing class %s %s...", colorize(self.id, "bold"), colorize("[%s]" % context, "cyan"))

            indent()
            tree = Parser.parse(self.getText(), self.id)
            ScopeScanner.scan(tree)
            outdent()

            self.project.getCache().store(field, tree, self.mtime, True)

        return tree

    def __getOptimizedTree(self, permutation=None, context=None):
        """Returns an optimized tree with permutations applied"""

        field = "opt-tree[%s]-%s" % (self.id, permutation)
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            tree = copy.deepcopy(self.__getTree("%s:plain" % context))

            # Logging
            msg = "Processing class %s" % colorize(self.id, "bold")
            if permutation:
                msg += colorize(" (%s)" % permutation, "grey")
            if context:
                msg += colorize(" [%s]" % context, "cyan")

            info("%s..." % msg)
            indent()

            # Apply permutation
            if permutation:
                jasy.js.clean.Permutate.patch(tree, permutation)

            # Cleanups
            jasy.js.clean.DeadCode.cleanup(tree)
            ScopeScanner.scan(tree)
            jasy.js.clean.Unused.cleanup(tree)

            self.project.getCache().store(field, tree, self.mtime, True)
            outdent()

        return tree

    def getDependencies(self, permutation=None, classes=None, warnings=True):
        """
        Returns a set of dependencies seen through the given list of known
        classes (ignoring all unknown items in original set). This method
        makes use of the meta data (see core/MetaData.py) and the variable data
        (see parse/ScopeData.py).
        """

        permutation = self.filterPermutation(permutation)

        meta = self.getMetaData(permutation)
        scope = self.getScopeData(permutation)

        result = set()

        # Manually defined names/classes
        for name in meta.requires:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.add(classes[name])
            elif warnings:
                warn("- Missing class (required): %s in %s", name, self.id)

        # Globally modified names (mostly relevant when working without namespaces)
        for name in scope.shared:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.add(classes[name])

        # Add classes from detected package access
        for package in scope.packages:
            if package in aliases:
                className = aliases[package]
                if className in classes:
                    result.add(classes[className])
                    continue

            orig = package
            while True:
                if package == self.id:
                    break

                elif package in classes and classes[package].kind == "class":
                    aliases[orig] = package
                    result.add(classes[package])
                    break

                else:
                    pos = package.rfind(".")
                    if pos == -1:
                        break

                    package = package[0:pos]

        # Manually excluded names/classes
        for name in meta.optionals:
            if name != self.id and name in classes and classes[name].kind == "class":
                result.remove(classes[name])
            elif warnings:
                warn("- Missing class (optional): %s in %s", name, self.id)

        return result

    def getScopeData(self, permutation=None):
        """
        Returns the top level scope object which contains information about the
        global variable and package usage/influence.
        """

        permutation = self.filterPermutation(permutation)

        field = "scope[%s]-%s" % (self.id, permutation)
        scope = self.project.getCache().read(field, self.mtime)
        if scope is None:
            scope = self.__getOptimizedTree(permutation, "scope").scope
            self.project.getCache().store(field, scope, self.mtime)

        return scope

    def getApi(self, highlight=True):
        field = "api[%s]-%s" % (self.id, highlight)
        apidata = self.project.getCache().read(field, self.mtime, inMemory=False)
        if apidata is None:
            apidata = ApiData(self.id, highlight)

            tree = self.__getTree(context="api")
            indent()
            apidata.scanTree(tree)
            outdent()

            metaData = self.getMetaData()
            apidata.addAssets(metaData.assets)
            for require in metaData.requires:
                apidata.addUses(require)
            for optional in metaData.optionals:
                apidata.removeUses(optional)

            apidata.addSize(self.getSize())
            apidata.addFields(self.getFields())

            self.project.getCache().store(field, apidata, self.mtime, inMemory=False)

        return apidata

    def getHighlightedCode(self):
        field = "highlighted[%s]" % self.id
        source = self.project.getCache().read(field, self.mtime)
        if source is None:
            if highlight is None:
                raise JasyError("Could not highlight JavaScript code! Please install Pygments.")

            lexer = JavascriptLexer(tabsize=2)
            formatter = HtmlFormatter(full=True, style="autumn", linenos="table", lineanchors="line")
            source = highlight(self.getText(), lexer, formatter)

            self.project.getCache().store(field, source, self.mtime)

        return source

    def getMetaData(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "meta[%s]-%s" % (self.id, permutation)
        meta = self.project.getCache().read(field, self.mtime)
        if meta is None:
            meta = MetaData(self.__getOptimizedTree(permutation, "meta"))
            self.project.getCache().store(field, meta, self.mtime)

        return meta

    def getFields(self):
        field = "fields[%s]" % (self.id)
        fields = self.project.getCache().read(field, self.mtime)
        if fields is None:
            fields = collectFields(self.__getTree(context="fields"))
            self.project.getCache().store(field, fields, self.mtime)

        return fields

    def getTranslations(self):
        field = "translations[%s]" % (self.id)
        result = self.project.getCache().read(field, self.mtime)
        if result is None:
            result = jasy.js.optimize.Translation.collectTranslations(self.__getTree(context="i18n"))
            self.project.getCache().store(field, result, self.mtime)

        return result

    def filterPermutation(self, permutation):
        if permutation:
            fields = self.getFields()
            if fields:
                return permutation.filter(fields)

        return None

    def getCompressed(self, permutation=None, translation=None, optimization=None, formatting=None, context="compressed"):
        permutation = self.filterPermutation(permutation)

        # Disable translation for caching / patching when not actually used
        if translation and not self.getTranslations():
            translation = None

        field = "compressed[%s]-%s-%s-%s-%s" % (self.id, permutation, translation, optimization, formatting)
        compressed = self.project.getCache().read(field, self.mtime)
        if compressed == None:
            tree = self.__getOptimizedTree(permutation, context)

            if translation or optimization:
                tree = copy.deepcopy(tree)

                if translation:
                    jasy.js.optimize.Translation.optimize(tree, translation)

                if optimization:
                    try:
                        optimization.apply(tree)
                    except jasy.js.output.Optimization.Error as error:
                        raise ClassError(self, "Could not compress class! %s" % error)

            compressed = Compressor(formatting).compress(tree)
            self.project.getCache().store(field, compressed, self.mtime)

        return compressed

    def getSize(self):
        field = "size[%s]" % self.id
        size = self.project.getCache().read(field, self.mtime)

        if size is None:
            compressed = self.getCompressed(context="size")
            optimized = self.getCompressed(permutation=defaultPermutation, optimization=defaultOptimization, context="size")
            zipped = zlib.compress(optimized.encode("utf-8"))

            size = {
                "compressed" : len(compressed),
                "optimized" : len(optimized),
                "zipped" : len(zipped)
            }

            self.project.getCache().store(field, size, self.mtime)

        return size

<|endoftext|>