<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2015 Intel Corporation.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of works must retain the original copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the original copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
#   may be used to endorse or promote products derived from this work without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
#         Fan, Yugang <yugang.fan@intel.com>
#         Hao, Yunfei <yunfeix.hao@intel.com>

import os
import shutil
import glob
import time
import sys
import stat
import random
import json
import logging
import zipfile
import signal
import fnmatch
import subprocess
import commands
import re
from optparse import OptionParser
import varshop
import utils

global LOG
LOG = utils.getLogger("build_android")

def packAPK(build_json=None, app_src=None, app_dest=None, app_name=None):
    BUILD_PARAMETERS = varshop.getValue("BUILD_PARAMETERS")
    BUILD_ROOT = varshop.getValue("BUILD_ROOT")
    BUILD_ROOT_SRC = varshop.getValue("BUILD_ROOT_SRC")
    BUILD_TIME= varshop.getValue("BUILD_TIME")
    CROSSWALK_VERSION = varshop.getValue("CROSSWALK_VERSION")
    DEFAULT_CMD_TIMEOUT= varshop.getValue("DEFAULT_CMD_TIMEOUT")
    PKG_MODES= varshop.getValue("PKG_MODES")
    PKG_ARCHS= varshop.getValue("PKG_ARCHS")
    app_name = app_name.replace("-", "_")
    get_real_arch = {"x86": "x86",
                     "x86_64": "x86_64",
                     "arm": "armeabi-v7a",
                     "arm64": "arm64-v8a"}

    #Use crosswalk zip in local mode
    #if not os.path.exists(os.path.join(BUILD_ROOT, "crosswalk-%s.zip" % CROSSWALK_VERSION)):
    #    if not utils.doCopy(
    #            os.path.join(BUILD_PARAMETERS.pkgpacktools, "crosswalk-%s.zip" % CROSSWALK_VERSION),
    #            os.path.join(BUILD_ROOT, "crosswalk-%s.zip" % CROSSWALK_VERSION)):
    #        return False

    files = glob.glob(os.path.join(BUILD_ROOT, "*.apk"))
    if files:
        if not utils.doRemove(files):
            return False

    ext_opt = []
    cmd_opt = ""
    url_opt = ""
    mode_opt = ""
    arch_opt = ""
    icon_opt = ""
    icons_opt = []
    version_opt = ""
    pkg_opt = ""
    version_code_opt = ""
    fullscreen_opt = ""
    orientation_opt = ""
    screenOn_opt = ""
    animatableView_opt = ""
    webp_opt = ""
    shortName_opt = ""
    permissions_opt = []

    common_opts = utils.safelyGetValue(build_json, "apk-common-opts")
    if common_opts is None:
        common_opts = "-r"
    else:
        common_opts_array = common_opts.split()
        if "-r" in common_opts_array:
            pass
        elif "--enable-remote-debugging" in common_opts_array:
            common_opts = common_opts.replace('--enable-remote-debugging', '')
        else:
            common_opts += "-r"

    tmp_opt = utils.safelyGetValue(build_json, "apk-ext-opt")
    if tmp_opt:
        ext_opt = [os.path.join(BUILD_ROOT_SRC, tmp_opt)]

    tmp_opt = utils.safelyGetValue(build_json, "apk-version-opt")
    if tmp_opt:
        version_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-fullscreen-opt")
    if tmp_opt:
        fullscreen_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-pkg-opt")
    if tmp_opt:
        pkg_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-cmd-opt")
    if tmp_opt:
        cmd_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-url-opt")
    if tmp_opt:
        url_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-orientation-opt")
    if tmp_opt:
        orientation_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-screenOn-opt")
    if tmp_opt:
        screenOn_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-animatableView-opt")
    if tmp_opt:
        animatableView_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-webp-opt")
    if tmp_opt:
        webp_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-shortName-opt")
    if tmp_opt:
        shortName_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-permissions-opt")
    if tmp_opt:
        permissions_opt = [tmp_opt]

    tmp_opt = utils.safelyGetValue(build_json, "apk-mode-opt")
    if tmp_opt:
        if tmp_opt in PKG_MODES:
            mode_opt = "--android=\"%s\"" % tmp_opt
            if tmp_opt == "embedded":
                mode_opt = ""
        else:
            LOG.error("Got wrong app mode: %s" % tmp_opt)
            return False
    else:
        mode_opt = "--android=\"%s\"" % BUILD_PARAMETERS.pkgmode
        if BUILD_PARAMETERS.pkgmode == "embedded":
            mode_opt = ""

    tmp_opt = utils.safelyGetValue(build_json, "apk-arch-opt")
    if tmp_opt:
        if tmp_opt in PKG_ARCHS:
            arch_opt = "%s" % tmp_opt
        else:
            LOG.error("Got wrong app arch: %s" % tmp_opt)
            return False
    else:
        arch_opt = "%s" % BUILD_PARAMETERS.pkgarch
    arch_opt = get_real_arch[arch_opt]

    tmp_opt = utils.safelyGetValue(build_json, "apk-icon-opt")
    if tmp_opt:
        icon_opt = "%s" % tmp_opt
        icon_set = {}
        icon_set["src"] = icon_opt
        icon_set["sizes"] = "72x72"
        icons_opt = [icon_set]
    elif tmp_opt == "":
        pass
    else:
        icon_opt = "icon.png"
        icon_set = {}
        icon_set["src"] = icon_opt
        icon_set["sizes"] = "72x72"
        icons_opt = [icon_set]

    manifest_opt = {}
    manifest_opt["name"] = "%s" % app_name
    if pkg_opt:
        manifest_opt["xwalk_package_id"] = "org.xwalk.%s" % pkg_opt
    else:
        manifest_opt["xwalk_package_id"] = "org.xwalk.%s" % app_name
    if url_opt:
        manifest_opt["start_url"] = url_opt
    else:
        manifest_opt["start_url"] = "index.html"
    if ext_opt:
        manifest_opt["xwalk_extensions"] = ext_opt
    if cmd_opt:
        manifest_opt["xwalk_command_line"] = cmd_opt
    if fullscreen_opt:
        manifest_opt["display"] = fullscreen_opt
    if version_opt:
        manifest_opt["xwalk_app_version"] = version_opt
    if icons_opt:
        manifest_opt["icons"] = icons_opt
    if orientation_opt:
        manifest_opt["orientation"] = orientation_opt
    if screenOn_opt:
        manifest_opt["xwalk_android_keep_screen_on"] = screenOn_opt
    if animatableView_opt:
        manifest_opt["xwalk_android_animatable_view"] = animatableView_opt
    if webp_opt:
        manifest_opt["xwalk_android_webp"] = webp_opt
    if shortName_opt:
        manifest_opt["short_name"] = shortName_opt
    if permissions_opt:
        manifest_opt["xwalk_android_permissions"] = permissions_opt

    manifest_opt = json.JSONEncoder().encode(manifest_opt)

    if utils.safelyGetValue(build_json, "apk-type") == "MANIFEST":
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, arch_opt, common_opts,
                       app_src)
    elif utils.safelyGetValue(build_json, "apk-type") == "HOSTEDAPP":
        if not url_opt:
            LOG.error(
                "Fail to find the key \"apk-url-opt\" for hosted APP packing")
            return False
        app_src_temp = "%s_%s" % (app_src, utils.getRandomStr())
        os.mkdir(app_src_temp)
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s --manifest='%s' " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, manifest_opt, arch_opt,
                       common_opts, app_src_temp)
    else:
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s --manifest='%s' " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, manifest_opt, arch_opt,
                       common_opts, app_src)

    orig_dir = os.getcwd()
    os.chdir(os.path.join(BUILD_ROOT))
    if not utils.doCMD(pack_cmd, DEFAULT_CMD_TIMEOUT):
        os.chdir(orig_dir)
        return False

    files = glob.glob(os.path.join(BUILD_ROOT, "*.apk"))
    if files:
        if not utils.doCopy(files[0], os.path.join(app_dest, "%s.apk" % app_name)):
            os.chdir(orig_dir)
            return False
    else:
        LOG.error("Fail to find the apk file")
        os.chdir(orig_dir)
        return False

    os.chdir(orig_dir)
    return True
<commit_msg>Remove icon_opts for subapp when apk type is HOSTEDAPP or MANIFESTAPP<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2015 Intel Corporation.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of works must retain the original copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the original copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
#   may be used to endorse or promote products derived from this work without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
#         Fan, Yugang <yugang.fan@intel.com>
#         Hao, Yunfei <yunfeix.hao@intel.com>

import os
import shutil
import glob
import time
import sys
import stat
import random
import json
import logging
import zipfile
import signal
import fnmatch
import subprocess
import commands
import re
from optparse import OptionParser
import varshop
import utils

global LOG
LOG = utils.getLogger("build_android")

def packAPK(build_json=None, app_src=None, app_dest=None, app_name=None):
    BUILD_PARAMETERS = varshop.getValue("BUILD_PARAMETERS")
    BUILD_ROOT = varshop.getValue("BUILD_ROOT")
    BUILD_ROOT_SRC = varshop.getValue("BUILD_ROOT_SRC")
    BUILD_TIME= varshop.getValue("BUILD_TIME")
    CROSSWALK_VERSION = varshop.getValue("CROSSWALK_VERSION")
    DEFAULT_CMD_TIMEOUT= varshop.getValue("DEFAULT_CMD_TIMEOUT")
    PKG_MODES= varshop.getValue("PKG_MODES")
    PKG_ARCHS= varshop.getValue("PKG_ARCHS")
    app_name = app_name.replace("-", "_")
    get_real_arch = {"x86": "x86",
                     "x86_64": "x86_64",
                     "arm": "armeabi-v7a",
                     "arm64": "arm64-v8a"}

    #Use crosswalk zip in local mode
    #if not os.path.exists(os.path.join(BUILD_ROOT, "crosswalk-%s.zip" % CROSSWALK_VERSION)):
    #    if not utils.doCopy(
    #            os.path.join(BUILD_PARAMETERS.pkgpacktools, "crosswalk-%s.zip" % CROSSWALK_VERSION),
    #            os.path.join(BUILD_ROOT, "crosswalk-%s.zip" % CROSSWALK_VERSION)):
    #        return False

    files = glob.glob(os.path.join(BUILD_ROOT, "*.apk"))
    if files:
        if not utils.doRemove(files):
            return False

    ext_opt = []
    cmd_opt = ""
    url_opt = ""
    mode_opt = ""
    arch_opt = ""
    icon_opt = ""
    icons_opt = []
    version_opt = ""
    pkg_opt = ""
    version_code_opt = ""
    fullscreen_opt = ""
    orientation_opt = ""
    screenOn_opt = ""
    animatableView_opt = ""
    webp_opt = ""
    shortName_opt = ""
    permissions_opt = []

    common_opts = utils.safelyGetValue(build_json, "apk-common-opts")
    if common_opts is None:
        common_opts = "-r"
    else:
        common_opts_array = common_opts.split()
        if "-r" in common_opts_array:
            pass
        elif "--enable-remote-debugging" in common_opts_array:
            common_opts = common_opts.replace('--enable-remote-debugging', '')
        else:
            common_opts += "-r"

    tmp_opt = utils.safelyGetValue(build_json, "apk-ext-opt")
    if tmp_opt:
        ext_opt = [os.path.join(BUILD_ROOT_SRC, tmp_opt)]

    tmp_opt = utils.safelyGetValue(build_json, "apk-version-opt")
    if tmp_opt:
        version_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-fullscreen-opt")
    if tmp_opt:
        fullscreen_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-pkg-opt")
    if tmp_opt:
        pkg_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-cmd-opt")
    if tmp_opt:
        cmd_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-url-opt")
    if tmp_opt:
        url_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-orientation-opt")
    if tmp_opt:
        orientation_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-screenOn-opt")
    if tmp_opt:
        screenOn_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-animatableView-opt")
    if tmp_opt:
        animatableView_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-webp-opt")
    if tmp_opt:
        webp_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-shortName-opt")
    if tmp_opt:
        shortName_opt = "%s" % tmp_opt

    tmp_opt = utils.safelyGetValue(build_json, "apk-permissions-opt")
    if tmp_opt:
        permissions_opt = [tmp_opt]

    tmp_opt = utils.safelyGetValue(build_json, "apk-mode-opt")
    if tmp_opt:
        if tmp_opt in PKG_MODES:
            mode_opt = "--android=\"%s\"" % tmp_opt
            if tmp_opt == "embedded":
                mode_opt = ""
        else:
            LOG.error("Got wrong app mode: %s" % tmp_opt)
            return False
    else:
        mode_opt = "--android=\"%s\"" % BUILD_PARAMETERS.pkgmode
        if BUILD_PARAMETERS.pkgmode == "embedded":
            mode_opt = ""

    tmp_opt = utils.safelyGetValue(build_json, "apk-arch-opt")
    if tmp_opt:
        if tmp_opt in PKG_ARCHS:
            arch_opt = "%s" % tmp_opt
        else:
            LOG.error("Got wrong app arch: %s" % tmp_opt)
            return False
    else:
        arch_opt = "%s" % BUILD_PARAMETERS.pkgarch
    arch_opt = get_real_arch[arch_opt]

    tmp_opt = utils.safelyGetValue(build_json, "apk-icon-opt")
    if tmp_opt:
        icon_opt = "%s" % tmp_opt
        icon_set = {}
        icon_set["src"] = icon_opt
        icon_set["sizes"] = "72x72"
        icons_opt = [icon_set]
    elif tmp_opt == "":
        pass
    else:
        icon_opt = "icon.png"
        icon_set = {}
        icon_set["src"] = icon_opt
        icon_set["sizes"] = "72x72"
        icons_opt = [icon_set]

    manifest_opt = {}
    manifest_opt["name"] = "%s" % app_name
    if pkg_opt:
        manifest_opt["xwalk_package_id"] = "org.xwalk.%s" % pkg_opt
    else:
        manifest_opt["xwalk_package_id"] = "org.xwalk.%s" % app_name
    if url_opt:
        manifest_opt["start_url"] = url_opt
    else:
        manifest_opt["start_url"] = "index.html"
    if ext_opt:
        manifest_opt["xwalk_extensions"] = ext_opt
    if cmd_opt:
        manifest_opt["xwalk_command_line"] = cmd_opt
    if fullscreen_opt:
        manifest_opt["display"] = fullscreen_opt
    if version_opt:
        manifest_opt["xwalk_app_version"] = version_opt
    if icons_opt and \
           utils.safelyGetValue(build_json, "apk-type") != "MANIFEST" and \
           utils.safelyGetValue(build_json, "apk-type") != "HOSTEDAPP":
        manifest_opt["icons"] = icons_opt
    if orientation_opt:
        manifest_opt["orientation"] = orientation_opt
    if screenOn_opt:
        manifest_opt["xwalk_android_keep_screen_on"] = screenOn_opt
    if animatableView_opt:
        manifest_opt["xwalk_android_animatable_view"] = animatableView_opt
    if webp_opt:
        manifest_opt["xwalk_android_webp"] = webp_opt
    if shortName_opt:
        manifest_opt["short_name"] = shortName_opt
    if permissions_opt:
        manifest_opt["xwalk_android_permissions"] = permissions_opt

    manifest_opt = json.JSONEncoder().encode(manifest_opt)

    if utils.safelyGetValue(build_json, "apk-type") == "MANIFEST":
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, arch_opt, common_opts,
                       app_src)
    elif utils.safelyGetValue(build_json, "apk-type") == "HOSTEDAPP":
        if not url_opt:
            LOG.error(
                "Fail to find the key \"apk-url-opt\" for hosted APP packing")
            return False
        app_src_temp = "%s_%s" % (app_src, utils.getRandomStr())
        os.mkdir(app_src_temp)
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s --manifest='%s' " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, manifest_opt, arch_opt,
                       common_opts, app_src_temp)
    else:
        pack_cmd = "crosswalk-pkg %s --crosswalk=%s --manifest='%s' " \
                   "-p android --targets=\"%s\" %s %s" % (
                       mode_opt, CROSSWALK_VERSION, manifest_opt, arch_opt,
                       common_opts, app_src)

    orig_dir = os.getcwd()
    os.chdir(os.path.join(BUILD_ROOT))
    if not utils.doCMD(pack_cmd, DEFAULT_CMD_TIMEOUT):
        os.chdir(orig_dir)
        return False

    files = glob.glob(os.path.join(BUILD_ROOT, "*.apk"))
    if files:
        if not utils.doCopy(files[0], os.path.join(app_dest, "%s.apk" % app_name)):
            os.chdir(orig_dir)
            return False
    else:
        LOG.error("Fail to find the apk file")
        os.chdir(orig_dir)
        return False

    os.chdir(orig_dir)
    return True
<|endoftext|>


<commit_before># Copyright 2016 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
from cStringIO import StringIO
from dateutil.parser import parse
from functools import partial
from gzip import GzipFile
import json
import logging
import math
from multiprocessing import cpu_count, Pool
import os
import tempfile
import time
import sqlite3

import boto3

from botocore.client import Config

log = logging.getLogger('c7n_traildb')

def dump(o):
    return json.dumps(o)

def load(s):
    return json.loads(s)

def chunks(iterable, size=50):
    """Break an iterable into lists of size"""
    batch = []
    for n in iterable:
        batch.append(n)
        if len(batch) % size == 0:
            yield batch
            batch = []
    if batch:
        yield batch

def process_trail_set(
        object_set, map_records, reduce_results=None, trail_bucket=None):
    s3 = boto3.Session().client('s3', config=Config(signature_version='s3v4'))
    previous = None
    for o in object_set:
        body = s3.get_object(Key=o['Key'], Bucket=trail_bucket)['Body']
        fh = GzipFile(fileobj=StringIO(body.read()))
        data = json.load(fh)
        s = map_records(data['Records'])
        if reduce_results:
            previous = reduce_results(s, previous)
    return previous

class TrailDB(object):

    def __init__(self, path):
        self.path = path
        self.conn = sqlite3.connect(self.path)
        self.cursor = self.conn.cursor()
        self._init()

    def _init(self):
        self.cursor.execute('''
           create table if not exists events (
              event_date   datetime,
              event_name   varchar(128),
              event_source varchar(128),
              user_agent   varchar(128),
              request_id   varchar(32),
              client_ip    varchar(32),
              user_id      varchar(128),
              error_code   varchar(256),
              error        text
        )''')
# omit due to size
#              response     text,
#              request      text,
#              user         text,

    def insert(self, records):
        self.cursor.executemany(
            "insert into events values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
            records)

    def flush(self):
        self.conn.commit()

def reduce_records(x, y):
    if y is None:
        return x
    elif x is None:
        return y
    y.extend(x)
    return y

# STOP = 42
#
# def store_records(output, q):
#    db = TrailDB(output)
#    while True:
#        results = q.get()
#        if results == STOP:
#            return
#        for r in results:
#            for fpath in r:
#                with open(fpath) as fh:
#                    db.insert(load(fh.read()))
#                os.remove(fpath)
#        db.flush()

def process_record_set(object_processor, q):
    def f(*args, **kw):
        r = object_processor(*args, **kw)
        if r:
            q.put(r)
        return r

def process_records(records,
                    uid_filter=None,
                    event_filter=None,
                    service_filter=None,
                    not_service_filter=None,
                    data_dir=None):

    user_records = []
    for r in records:
        if not_service_filter and r['eventSource'] == not_service_filter:
            continue

        utype = r['userIdentity'].get('type', None)
        if utype == 'Root':
            uid = 'root'
        elif utype == 'SAMLUser':
            uid = r['userIdentity']['userName']
        elif utype is None and r['userIdentity']['invokedBy'] == 'AWS Internal':
            uid = r['userIdentity']['invokedBy']
        else:
            uid = r['userIdentity'].get('arn', '')

        if uid_filter and uid_filter not in uid.lower():
            continue
        elif event_filter and not r['eventName'] == event_filter:
            continue
        elif service_filter and not r['eventSource'] == service_filter:
            continue
        user_records.append((
            r['eventTime'],
            r['eventName'],
            r['eventSource'],
            r.get('userAgent', ''),
            r.get('requestID', ''),
            r.get('sourceIPAddress', ''),
            uid,
            # TODO make this optional, for now omit for size
            #            json.dumps(r['requestParameters']),
            #            json.dumps(r['responseElements']),
            #            json.dumps(r['userIdentity']),
            r.get('errorCode', None),
            r.get('errorMessage', None)
        ))
    if data_dir:
        if not user_records:
            return
        # Spool to temporary files to get out of mem
        fh = tempfile.NamedTemporaryFile(dir=data_dir, delete=False)
        fh.write(dump(user_records))
        fh.flush()
        fh.close()
        return [fh.name]
    return user_records

def process_bucket(
        bucket_name, prefix,
        output=None, uid_filter=None, event_filter=None,
        service_filter=None, not_service_filter=None, data_dir=None):

    s3 = boto3.Session().client(
        's3', config=Config(signature_version='s3v4'))

    paginator = s3.get_paginator('list_objects')
    # PyPy has some memory leaks.... :-(
    pool = Pool(maxtasksperchild=10)
    t = time.time()
    object_count = object_size = 0

    log.info("Processing:%d cloud-trail %s" % (
        cpu_count(),
        prefix))

    record_processor = partial(
        process_records,
        uid_filter=uid_filter,
        event_filter=event_filter,
        service_filter=service_filter,
        not_service_filter=not_service_filter,
        data_dir=data_dir)

    object_processor = partial(
        process_trail_set,
        map_records=record_processor,
        reduce_results=reduce_records,
        trail_bucket=bucket_name)
    db = TrailDB(output)

    bsize = math.ceil(1000 / float(cpu_count()))
    for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix):
        objects = page.get('Contents', ())
        object_count += len(objects)
        object_size += sum([o['Size'] for o in objects])

        pt = time.time()
        if pool:
            results = pool.map(object_processor, chunks(objects, bsize))
        else:
            results = map(object_processor, chunks(objects, bsize))

        st = time.time()
        log.info("Loaded page time:%0.2fs", st - pt)

        for r in results:
            for fpath in r:
                with open(fpath) as fh:
                    db.insert(load(fh.read()))
                os.remove(fpath)
            db.flush()

        l = t
        t = time.time()

        log.info("Stored page time:%0.2fs", t - st)
        log.info(
            "Processed paged time:%0.2f size:%s count:%s" % (
                t - l, object_size, object_count))
        if objects:
            log.info('Last Page Key: %s', objects[-1]['Key'])

def get_bucket_path(options):
    prefix = "AWSLogs/%(account)s/CloudTrail/%(region)s/" % {
        'account': options.account, 'region': options.region}
    if options.prefix:
        prefix = "%s/%s" % (options.prefix.strip('/'), prefix)
    if options.day:
        date = parse(options.day)
        date_prefix = date.strftime("%Y/%m/%d/")
    if options.month:
        date = parse(options.month)
        date_prefix = date.strftime("%Y/%m/")
    if date_prefix:
        prefix += date_prefix
    return prefix

def setup_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("--bucket", required=True)
    parser.add_argument("--prefix", default="")
    parser.add_argument("--account", required=True)
    parser.add_argument("--user")
    parser.add_argument("--event")
    parser.add_argument("--source")
    parser.add_argument("--not-source")
    parser.add_argument("--day")
    parser.add_argument("--month")
    parser.add_argument("--tmpdir", default="/tmp/traildb")
    parser.add_argument("--region", default="us-east-1")
    parser.add_argument("--output", default="results.db")
    return parser

def main():
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('botocore').setLevel(logging.WARNING)
    parser = setup_parser()
    options = parser.parse_args()

    if options.tmpdir and not os.path.exists(options.tmpdir):
        os.makedirs(options.tmpdir)
    prefix = get_bucket_path(options)

    process_bucket(
        options.bucket,
        prefix,
        options.output,
        options.user,
        options.event,
        options.source,
        options.not_source,
        options.tmpdir
    )

if __name__ == '__main__':
    main()
<commit_msg>tools/c7n_traildb  cli flag to add optional fields to trail records table (#1412)<commit_after># Copyright 2016 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
from cStringIO import StringIO
from dateutil.parser import parse
from functools import partial
from gzip import GzipFile
import json
import logging
import math
from multiprocessing import cpu_count, Pool
from c7n.credentials import assumed_session, SessionFactory
import os
import tempfile
import time
import sqlite3

import boto3

from botocore.client import Config

log = logging.getLogger('c7n_traildb')

options = None

def dump(o):
    return json.dumps(o)

def load(s):
    return json.loads(s)

def chunks(iterable, size=50):
    """Break an iterable into lists of size"""
    batch = []
    for n in iterable:
        batch.append(n)
        if len(batch) % size == 0:
            yield batch
            batch = []
    if batch:
        yield batch

def process_trail_set(
        object_set, map_records, reduce_results=None, trail_bucket=None):

    session_factory = SessionFactory(
        options.region, options.profile, options.assume_role)

    s3 = session_factory().client(
        's3', config=Config(signature_version='s3v4'))

    previous = None
    for o in object_set:
        body = s3.get_object(Key=o['Key'], Bucket=trail_bucket)['Body']
        fh = GzipFile(fileobj=StringIO(body.read()))
        data = json.load(fh)
        s = map_records(data['Records'])
        if reduce_results:
            previous = reduce_results(s, previous)
    return previous

class TrailDB(object):

    def __init__(self, path):
        self.path = path
        self.conn = sqlite3.connect(self.path)
        self.cursor = self.conn.cursor()
        self._init()

    def _init(self):
        command = '''
           create table if not exists events (
              event_date   datetime,
              event_name   varchar(128),
              event_source varchar(128),
              user_agent   varchar(128),
              request_id   varchar(32),
              client_ip    varchar(32),
              user_id      varchar(128),
              error_code   varchar(256),
              error        text'''

        if options.field:
            for field in options.field:
                command += ",\n{}    text".format(field)

        command += ')'
        self.cursor.execute(command)

    def insert(self, records):
        command = "insert into events values (?, ?, ?, ?, ?, ?, ?, ?, ?"

        if options.field:
            command += ', ?' * len(options.field)

        command += ")"
        self.cursor.executemany(command, records)

    def flush(self):
        self.conn.commit()

def reduce_records(x, y):
    if y is None:
        return x
    elif x is None:
        return y
    y.extend(x)
    return y

# STOP = 42
#
# def store_records(output, q):
#    db = TrailDB(output)
#    while True:
#        results = q.get()
#        if results == STOP:
#            return
#        for r in results:
#            for fpath in r:
#                with open(fpath) as fh:
#                    db.insert(load(fh.read()))
#                os.remove(fpath)
#        db.flush()

def process_record_set(object_processor, q):
    def f(*args, **kw):
        r = object_processor(*args, **kw)
        if r:
            q.put(r)
        return r

def process_records(records,
                    uid_filter=None,
                    event_filter=None,
                    service_filter=None,
                    not_service_filter=None,
                    data_dir=None):

    user_records = []
    for r in records:
        if not_service_filter and r['eventSource'] == not_service_filter:
            continue

        utype = r['userIdentity'].get('type', None)
        if utype == 'Root':
            uid = 'root'
        elif utype == 'SAMLUser':
            uid = r['userIdentity']['userName']
        elif utype is None and r['userIdentity']['invokedBy'] == 'AWS Internal':
            uid = r['userIdentity']['invokedBy']
        else:
            uid = r['userIdentity'].get('arn', '')

        if uid_filter and uid_filter not in uid.lower():
            continue
        elif event_filter and not r['eventName'] == event_filter:
            continue
        elif service_filter and not r['eventSource'] == service_filter:
            continue

        user_record = (
            r['eventTime'],
            r['eventName'],
            r['eventSource'],
            r.get('userAgent', ''),
            r.get('requestID', ''),
            r.get('sourceIPAddress', ''),
            uid,
            r.get('errorCode', None),
            r.get('errorMessage', None)
        )

        # Optional data can be added to each record.
        # Field names are Case Sensitive.
        if options.field:
            for field in options.field:
                user_record += (json.dumps(r[field]), )

        user_records.append(user_record)

    if data_dir:
        if not user_records:
            return
        # Spool to temporary files to get out of mem
        fh = tempfile.NamedTemporaryFile(dir=data_dir, delete=False)
        fh.write(dump(user_records))
        fh.flush()
        fh.close()
        return [fh.name]
    return user_records

def process_bucket(
        bucket_name, prefix,
        output=None, uid_filter=None, event_filter=None,
        service_filter=None, not_service_filter=None, data_dir=None):

    session_factory = SessionFactory(
        options.region, options.profile, options.assume_role)

    s3 = session_factory().client(
        's3', config=Config(signature_version='s3v4'))

    paginator = s3.get_paginator('list_objects')
    # PyPy has some memory leaks.... :-(
    pool = Pool(maxtasksperchild=10)
    t = time.time()
    object_count = object_size = 0

    log.info("Processing:%d cloud-trail %s" % (
        cpu_count(),
        prefix))

    record_processor = partial(
        process_records,
        uid_filter=uid_filter,
        event_filter=event_filter,
        service_filter=service_filter,
        not_service_filter=not_service_filter,
        data_dir=data_dir)

    object_processor = partial(
        process_trail_set,
        map_records=record_processor,
        reduce_results=reduce_records,
        trail_bucket=bucket_name)
    db = TrailDB(output)

    bsize = math.ceil(1000 / float(cpu_count()))
    for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix):
        objects = page.get('Contents', ())
        object_count += len(objects)
        object_size += sum([o['Size'] for o in objects])

        pt = time.time()
        if pool:
            results = pool.map(object_processor, chunks(objects, bsize))
        else:
            results = map(object_processor, chunks(objects, bsize))

        st = time.time()
        log.info("Loaded page time:%0.2fs", st - pt)

        for r in results:
            for fpath in r:
                with open(fpath) as fh:
                    db.insert(load(fh.read()))
                os.remove(fpath)
            db.flush()

        l = t
        t = time.time()

        log.info("Stored page time:%0.2fs", t - st)
        log.info(
            "Processed paged time:%0.2f size:%s count:%s" % (
                t - l, object_size, object_count))
        if objects:
            log.info('Last Page Key: %s', objects[-1]['Key'])

def get_bucket_path(options):
    prefix = "AWSLogs/%(account)s/CloudTrail/%(region)s/" % {
        'account': options.account, 'region': options.region}
    if options.prefix:
        prefix = "%s/%s" % (options.prefix.strip('/'), prefix)
    if options.day:
        date = parse(options.day)
        date_prefix = date.strftime("%Y/%m/%d/")
    if options.month:
        date = parse(options.month)
        date_prefix = date.strftime("%Y/%m/")
    if date_prefix:
        prefix += date_prefix
    return prefix

def setup_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("--bucket", required=True)
    parser.add_argument("--prefix", default="")
    parser.add_argument("--account", required=True)
    parser.add_argument("--user")
    parser.add_argument("--event")
    parser.add_argument("--source")
    parser.add_argument("--not-source")
    parser.add_argument("--day")
    parser.add_argument("--month")
    parser.add_argument("--tmpdir", default="/tmp/traildb")
    parser.add_argument("--region", default="us-east-1")
    parser.add_argument("--output", default="results.db")
    parser.add_argument(
        "--profile", default=os.environ.get('AWS_PROFILE'),
        help="AWS Account Config File Profile to utilize")
    parser.add_argument(
        "--assume", default=None, dest="assume_role",
        help="Role to assume")
    parser.add_argument('--field', action='append',
        help='additonal fields that can be added to each record',
        choices=['userIdentity', 'requestParameters', 'responseElements'])
    return parser

def main():
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('botocore').setLevel(logging.WARNING)
    global options
    parser = setup_parser()
    options = parser.parse_args()

    if options.tmpdir and not os.path.exists(options.tmpdir):
        os.makedirs(options.tmpdir)
    prefix = get_bucket_path(options)

    process_bucket(
        options.bucket,
        prefix,
        options.output,
        options.user,
        options.event,
        options.source,
        options.not_source,
        options.tmpdir
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import shutil
import time
import requests

def download_dataset_subset(start, end, id_file_path, images_dir, descs_dir, thread_id):
    # Announce the start of the thread (mainly for debugging)
    print('Thread {0} started'.format(thread_id))

    # The url template is 'base_url_prefix - image_id - base_url_suffix'
    base_url_prefix = 'https://isic-archive.com/api/v1/image/'
    base_url_suffix = '/download?contentDisposition=inline'

    with open(id_file_path) as f:
        # get all the ids
        ids = f.readlines()

        # for each id:
        # 1. get the corresponding image and description urls
        # 2. download the image and the description from the urls
        for index, id in enumerate(ids[start:end]):
            # Update the index to the right offset of the bin we're working on
            index = index + start

            print("downloading image ({0}/{1})".format(index, end))
            # strip the id from the \n at the end
            id = id.strip()

            # get the corresponding image url
            url_image = base_url_prefix + id + base_url_suffix
            # get the corresponding description url
            url_desc = base_url_prefix + id
            print ("url_image = {0}".format(url_image))

            # Download the image and desctiprion using the urls
            # Sometimes their site isn't responding well, and than an error occurs,
            # So we will retry the same url 5 seconds later, until it succeeds
            succeeded = False
            while not succeeded:
                try:
                    response_image = requests.get(url_image, stream=True)
                    response_desc = requests.get(url_desc, stream=True)
                    succeeded = True
                except:
                    print "trying again index number ", index
                    time.sleep(5)

            # Write the image into a file
            imgFilePath = os.path.join(images_dir, 'ISIC_{0}.jpg'.format(str(index).zfill(7)))
            with open(imgFilePath, 'wb') as imageFile:
                shutil.copyfileobj(response_image.raw, imageFile)

            # Parse the description and write the description into a file
            parsed_desc = parse_desc(response_desc.content)
            descFilePath = os.path.join(descs_dir, 'ISIC_{0}'.format(str(index).zfill(7)))
            with open(descFilePath, 'w') as descFile:
                for line in parsed_desc:
                    descFile.write("%s\n" % line)

def parse_desc(desc):
    line_start_index = 0
    line_end_index = 0
    lines = []
    indentation = 0

    while line_end_index < len(desc):
        if desc[line_end_index:line_end_index+2] == '},':
            line = desc[line_start_index: line_end_index]
            # indent the line
            line = ' ' * indentation + line
            lines.append(line)
            line_start_index = line_end_index + 2
            line_end_index = line_start_index
            # update indentation
            indentation -= 4

        elif desc[line_end_index] in ['{', '}', ',', '[', ']']:
            line = desc[line_start_index: line_end_index + 1]
            # indent the line
            line = ' '*indentation + line
            lines.append(line)

            # update indentation
            if desc[line_end_index] in ['{', '[']:
                indentation += 4
            elif desc[line_end_index] in ['}', ']']:
                indentation -= 4

            # update pointers
            line_start_index = line_end_index + 1
            line_end_index = line_start_index

        else:
            line_end_index += 1

    return lines

<commit_msg>Fixed parser by kwuth<commit_after>import os
import shutil
import time
import requests

def download_dataset_subset(start, end, id_file_path, images_dir, descs_dir, thread_id):
    # Announce the start of the thread (mainly for debugging)
    print('Thread {0} started'.format(thread_id))

    # The url template is 'base_url_prefix - image_id - base_url_suffix'
    base_url_prefix = 'https://isic-archive.com/api/v1/image/'
    base_url_suffix = '/download?contentDisposition=inline'

    with open(id_file_path) as f:
        # get all the ids
        ids = f.readlines()

        # for each id:
        # 1. get the corresponding image and description urls
        # 2. download the image and the description from the urls
        for index, id in enumerate(ids[start:end]):
            # Update the index to the right offset of the bin we're working on
            index = index + start

            print("downloading image ({0}/{1})".format(index, end))
            # strip the id from the \n at the end
            id = id.strip()

            # get the corresponding image url
            url_image = base_url_prefix + id + base_url_suffix
            # get the corresponding description url
            url_desc = base_url_prefix + id
            print ("url_image = {0}".format(url_image))

            # Download the image and desctiprion using the urls
            # Sometimes their site isn't responding well, and than an error occurs,
            # So we will retry the same url 5 seconds later, until it succeeds
            succeeded = False
            while not succeeded:
                try:
                    response_image = requests.get(url_image, stream=True)
                    response_desc = requests.get(url_desc, stream=True)
                    succeeded = True
                except:
                    print "trying again index number ", index
                    time.sleep(5)

            # Write the image into a file
            imgFilePath = os.path.join(images_dir, 'ISIC_{0}.jpg'.format(str(index).zfill(7)))
            with open(imgFilePath, 'wb') as imageFile:
                shutil.copyfileobj(response_image.raw, imageFile)

            # Parse the description and write the description into a file
            parsed_desc = parse_desc(response_desc.content)
            descFilePath = os.path.join(descs_dir, 'ISIC_{0}'.format(str(index).zfill(7)))
            with open(descFilePath, 'w') as descFile:
                for line in parsed_desc:
                    descFile.write("%s\n" % line)

def parse_desc(desc):
    line_start_index = 0
    line_end_index = 0
    lines = []
    indentation = 0

    while line_end_index < len(desc):
        if desc[line_end_index:line_end_index+2] == '},':
            line = desc[line_start_index: line_end_index + 2]
            # indent the line
            line = ' ' * indentation + line
            lines.append(line)
            line_start_index = line_end_index + 2
            line_end_index = line_start_index
            # update indentation
            indentation -= 4

        elif desc[line_end_index] in ['{', '}', ',', '[', ']']:
            line = desc[line_start_index: line_end_index + 1]
            # indent the line
            line = ' '*indentation + line
            lines.append(line)

            # update indentation
            if desc[line_end_index] in ['{', '[']:
                indentation += 4
            elif desc[line_end_index] in ['}', ']']:
                indentation -= 4

            # update pointers
            line_start_index = line_end_index + 1
            line_end_index = line_start_index

        else:
            line_end_index += 1

    return lines

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Makes sure that all EXE and DLL files in the provided directory were built
correctly.

In essense it runs a subset of BinScope tests ensuring that binaries have
/NXCOMPAT, /DYNAMICBASE and /SAFESEH.
"""

import os
import optparse
import sys

# Find /third_party/pefile based on current directory and script path.
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..',
                             'third_party', 'pefile'))
import pefile

PE_FILE_EXTENSIONS = ['.exe', '.dll']
DYNAMICBASE_FLAG = 0x0040
NXCOMPAT_FLAG = 0x0100
NO_SEH_FLAG = 0x0400
MACHINE_TYPE_AMD64 = 0x8664

# Please do not add your file here without confirming that it indeed doesn't
# require /NXCOMPAT and /DYNAMICBASE.  Contact cpu@chromium.org or your local
# Windows guru for advice.
EXCLUDED_FILES = ['chrome_frame_mini_installer.exe',
                  'mini_installer.exe',
                  'wow_helper.exe']

def IsPEFile(path):
  return (os.path.isfile(path) and
          os.path.splitext(path)[1].lower() in PE_FILE_EXTENSIONS and
          os.path.basename(path) not in EXCLUDED_FILES)

def main(options, args):
  directory = args[0]
  pe_total = 0
  pe_passed = 0

  for file in os.listdir(directory):
    path = os.path.abspath(os.path.join(directory, file))
    if not IsPEFile(path):
      continue
    pe = pefile.PE(path, fast_load=True)
    pe.parse_data_directories(directories=[
        pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']])
    pe_total = pe_total + 1
    success = True

    # Check for /DYNAMICBASE.
    if pe.OPTIONAL_HEADER.DllCharacteristics & DYNAMICBASE_FLAG:
      if options.verbose:
        print "Checking %s for /DYNAMICBASE... PASS" % path
    else:
      success = False
      print "Checking %s for /DYNAMICBASE... FAIL" % path

    # Check for /NXCOMPAT.
    if pe.OPTIONAL_HEADER.DllCharacteristics & NXCOMPAT_FLAG:
      if options.verbose:
        print "Checking %s for /NXCOMPAT... PASS" % path
    else:
      success = False
      print "Checking %s for /NXCOMPAT... FAIL" % path

    # Check for /SAFESEH. Binaries should meet one of the following
    # criteria:
    #   1) Have no SEH table as indicated by the DLL characteristics
    #   2) Have a LOAD_CONFIG section containing a valid SEH table
    #   3) Be a 64-bit binary, in which case /SAFESEH isn't required
    #
    # Refer to the following MSDN article for more information:
    # http://msdn.microsoft.com/en-us/library/9a89h429.aspx
    if (pe.OPTIONAL_HEADER.DllCharacteristics & NO_SEH_FLAG or
        (hasattr(pe, "DIRECTORY_ENTRY_LOAD_CONFIG") and
         pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SEHandlerCount > 0 and
         pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SEHandlerTable != 0) or
        pe.FILE_HEADER.Machine == MACHINE_TYPE_AMD64):
      if options.verbose:
        print "Checking %s for /SAFESEH... PASS" % path
    else:
      success = False
      print "Checking %s for /SAFESEH... FAIL" % path

    # Update tally.
    if success:
      pe_passed = pe_passed + 1

  print "Result: %d files found, %d files passed" % (pe_total, pe_passed)
  if pe_passed != pe_total:
    sys.exit(1)

if __name__ == '__main__':
  usage = "Usage: %prog [options] DIRECTORY"
  option_parser = optparse.OptionParser(usage=usage)
  option_parser.add_option("-v", "--verbose", action="store_true",
                           default=False, help="Print debug logging")
  options, args = option_parser.parse_args()
  if not args:
    option_parser.print_help()
    sys.exit(0)
  main(options, args)
<commit_msg>Add xinput1_3.dll to checkbins' excluded_files<commit_after>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Makes sure that all EXE and DLL files in the provided directory were built
correctly.

In essense it runs a subset of BinScope tests ensuring that binaries have
/NXCOMPAT, /DYNAMICBASE and /SAFESEH.
"""

import os
import optparse
import sys

# Find /third_party/pefile based on current directory and script path.
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..',
                             'third_party', 'pefile'))
import pefile

PE_FILE_EXTENSIONS = ['.exe', '.dll']
DYNAMICBASE_FLAG = 0x0040
NXCOMPAT_FLAG = 0x0100
NO_SEH_FLAG = 0x0400
MACHINE_TYPE_AMD64 = 0x8664

# Please do not add your file here without confirming that it indeed doesn't
# require /NXCOMPAT and /DYNAMICBASE.  Contact cpu@chromium.org or your local
# Windows guru for advice.
EXCLUDED_FILES = ['chrome_frame_mini_installer.exe',
                  'mini_installer.exe',
                  'wow_helper.exe',
                  'xinput1_3.dll' # Microsoft DirectX redistributable.
                  ]

def IsPEFile(path):
  return (os.path.isfile(path) and
          os.path.splitext(path)[1].lower() in PE_FILE_EXTENSIONS and
          os.path.basename(path) not in EXCLUDED_FILES)

def main(options, args):
  directory = args[0]
  pe_total = 0
  pe_passed = 0

  for file in os.listdir(directory):
    path = os.path.abspath(os.path.join(directory, file))
    if not IsPEFile(path):
      continue
    pe = pefile.PE(path, fast_load=True)
    pe.parse_data_directories(directories=[
        pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']])
    pe_total = pe_total + 1
    success = True

    # Check for /DYNAMICBASE.
    if pe.OPTIONAL_HEADER.DllCharacteristics & DYNAMICBASE_FLAG:
      if options.verbose:
        print "Checking %s for /DYNAMICBASE... PASS" % path
    else:
      success = False
      print "Checking %s for /DYNAMICBASE... FAIL" % path

    # Check for /NXCOMPAT.
    if pe.OPTIONAL_HEADER.DllCharacteristics & NXCOMPAT_FLAG:
      if options.verbose:
        print "Checking %s for /NXCOMPAT... PASS" % path
    else:
      success = False
      print "Checking %s for /NXCOMPAT... FAIL" % path

    # Check for /SAFESEH. Binaries should meet one of the following
    # criteria:
    #   1) Have no SEH table as indicated by the DLL characteristics
    #   2) Have a LOAD_CONFIG section containing a valid SEH table
    #   3) Be a 64-bit binary, in which case /SAFESEH isn't required
    #
    # Refer to the following MSDN article for more information:
    # http://msdn.microsoft.com/en-us/library/9a89h429.aspx
    if (pe.OPTIONAL_HEADER.DllCharacteristics & NO_SEH_FLAG or
        (hasattr(pe, "DIRECTORY_ENTRY_LOAD_CONFIG") and
         pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SEHandlerCount > 0 and
         pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SEHandlerTable != 0) or
        pe.FILE_HEADER.Machine == MACHINE_TYPE_AMD64):
      if options.verbose:
        print "Checking %s for /SAFESEH... PASS" % path
    else:
      success = False
      print "Checking %s for /SAFESEH... FAIL" % path

    # Update tally.
    if success:
      pe_passed = pe_passed + 1

  print "Result: %d files found, %d files passed" % (pe_total, pe_passed)
  if pe_passed != pe_total:
    sys.exit(1)

if __name__ == '__main__':
  usage = "Usage: %prog [options] DIRECTORY"
  option_parser = optparse.OptionParser(usage=usage)
  option_parser.add_option("-v", "--verbose", action="store_true",
                           default=False, help="Print debug logging")
  options, args = option_parser.parse_args()
  if not args:
    option_parser.print_help()
    sys.exit(0)
  main(options, args)
<|endoftext|>


<commit_before>import sys

if sys.version_info[0] >= 3:
    appName = "DrawBotPy3"
else:
    appName = "DrawBot"

__version__ = "3.102"
<commit_msg>touching version number<commit_after>import sys

if sys.version_info[0] >= 3:
    appName = "DrawBotPy3"
else:
    appName = "DrawBot"

__version__ = "3.104"
<|endoftext|>


<commit_before># Copyright 2013-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/env python

"""
p4c - P4 Compiler Driver

"""

from __future__ import absolute_import
import argparse
import glob
import os
import sys
import re

import p4c_src.config as config
import p4c_src

# \TODO: let the backends set their versions ...
p4c_version = p4c_src.__version__

def set_version(ver):
    global p4c_version
    p4c_version = ver

def get_version():
    return p4c_version

def display_supported_targets(cfg):
    ret = "Supported targets in \"target, arch\" tuple:\n"
    for target in cfg.target:
        ret += str(target) + "\n"
    return ret

def add_developer_options(parser):
    parser.add_argument("-T", dest="log_levels",
                        action="append", default=[],
                        help="[Compiler debugging] Adjust logging level per file (see below)")
    parser.add_argument("--top4", dest="passes",
                        action="append", default=[],
                        help="[Compiler debugging] Dump the P4 representation after \
                               passes whose name contains one of `passX' substrings. \
                               When '-v' is used this will include the compiler IR.")
    parser.add_argument("--dump", dest="dump_dir", default=None,
                        help="[Compiler debugging] Folder where P4 programs are dumped.")
    parser.add_argument("--toJson", dest="json", default=None,
                        help="Dump IR to JSON in the specified file.")
    parser.add_argument("--pp", dest="pretty_print", default=None,
                        help="Pretty-print the program in the specified file.")

def main():
    parser = argparse.ArgumentParser(conflict_handler='resolve')
    parser.add_argument("-V", "--version", dest="show_version",
                        help="show version and exit",
                        action="store_true", default=False)
    parser.add_argument("-v", "--debug", dest="debug",
                        help="verbose",
                        action="store_true", default=False)
    parser.add_argument("-###", "--test-only", dest="dry_run",
                        help="print (but do not run) the commands",
                        action="store_true", default=False)
    parser.add_argument("-Xpreprocessor", dest="preprocessor_options",
                        metavar="<arg>",
                        help="Pass <arg> to the preprocessor",
                        action="append", default=[])
    parser.add_argument("-Xp4c", dest="compiler_options",
                        metavar="<arg>",
                        help="Pass <arg> to the compiler",
                        action="append", default=[])
    parser.add_argument("-Xassembler", dest="assembler_options",
                        metavar="<arg>",
                        help="Pass <arg> to the assembler",
                        action="append", default=[])
    parser.add_argument("-Xlinker", dest="linker_options",
                        metavar="<arg>",
                        help="Pass <arg> to the linker",
                        action="append", default=[])
    parser.add_argument("-b", "--target", dest="target",
                        help="specify target device",
                        action="store", default="bmv2")
    parser.add_argument("-a", "--arch", dest="arch",
                        help="specify target architecture",
                        action="store", default="v1model")
    parser.add_argument("-c", dest="run_all",
                        help="Only run preprocess, compile, and assemble steps",
                        action="store_true", default=True)
    parser.add_argument("-D", dest="preprocessor_defines",
                        help="define a macro to be used by the preprocessor",
                        action="append", default=[])
    parser.add_argument("-E", dest="run_preprocessor_only",
                        help="Only run the preprocessor",
                        action="store_true", default=False)
    parser.add_argument("-e", dest="skip_preprocessor",
                        help="Skip the preprocessor",
                        action="store_true", default=False)
    parser.add_argument("-g", dest="debug_info",
                        help="Generate debug information",
                        action="store_true", default=False)
    parser.add_argument("-I", dest="search_path",
                        help="Add directory to include search path",
                        action="append", default=[])
    parser.add_argument("-o", dest="output_directory",
                        help="Write output to the provided path",
                        action="store", metavar="PATH", default=".")
    parser.add_argument("--p4runtime-file",
                        help="Write a P4Runtime control plane API description "
                        "to the specified file.",
                        action="store", default=None)
    parser.add_argument("--p4runtime-format",
                        choices=["binary", "json", "text"],
                        help="Choose output format for the P4Runtime API "
                        "description (default is binary).",
                        action="store", default="binary")
    parser.add_argument("--target-help", dest="show_target_help",
                        help="Display target specific command line options.",
                        action="store_true", default=False)
    parser.add_argument("-S", dest="run_till_assembler",
                        help="Only run the preprocess and compilation steps",
                        action="store_true", default=False)
    parser.add_argument("--std", "-x", dest="language",
                        choices = ["p4-14", "p4-16"],
                        help="Treat subsequent input files as having type language.",
                        action="store", default="p4-16")

    if (os.environ['P4C_BUILD_TYPE'] == "DEVELOPER"):
        add_developer_options(parser)

    parser.add_argument("source_file", nargs='?', help="Files to compile", default=None)

    # load supported configuration.
    # We load these before we parse options, so that backends can register
    # proprietary options
    cfg_files = glob.glob("{}/*.cfg".format(os.environ['P4C_CFG_PATH']))
    cfg = config.Config(config_prefix = "p4c")
    for cf in cfg_files:
        cfg.load_from_config(cf, parser)

    # parse the arguments
    opts = parser.parse_args()

    user_defined_version = os.environ.get('P4C_DEFAULT_VERSION')
    if user_defined_version != None:
        opts.language = user_defined_version

    user_defined_target = os.environ.get('P4C_DEFAULT_TARGET')
    if user_defined_target != None:
        opts.target = user_defined_target

    user_defined_arch = os.environ.get('P4C_DEFAULT_ARCH')
    if user_defined_arch != None:
        opts.arch = user_defined_arch

    # deal with early exits
    if opts.show_version:
        print "p4c", get_version()
        sys.exit(0)

    if opts.show_target_help:
        print display_supported_targets(cfg)
        sys.exit(0)

    if not opts.source_file:
        parser.error('No input specified.')

    # check that the tuple value is correct
    backend = (opts.target, opts.arch)
    if (len(backend) != 2):
        parser.error("Invalid target and arch tuple: {}\n{}".\
                     format(backend, display_supported_targets(cfg)))

    # find the backend
    backend = None
    for target in cfg.target:
        regex = target._backend.replace('*', '[a-zA-Z0-9*]*')
        pattern = re.compile(regex)
        if (pattern.match(opts.target + '-' + opts.arch)):
            backend = target
            break
    if backend == None:
        parser.error("Unknown backend: {}-{}".format(str(opts.target), str(opts.arch)))

    # set all configuration and command line options for backend
    backend.process_command_line_options(opts)
    # run all commands
    rc = backend.run()
    sys.exit(rc)
<commit_msg>fail early when the input file does not exist<commit_after># Copyright 2013-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/env python

"""
p4c - P4 Compiler Driver

"""

from __future__ import absolute_import
import argparse
import glob
import os
import sys
import re

import p4c_src.config as config
import p4c_src

# \TODO: let the backends set their versions ...
p4c_version = p4c_src.__version__

def set_version(ver):
    global p4c_version
    p4c_version = ver

def get_version():
    return p4c_version

def display_supported_targets(cfg):
    ret = "Supported targets in \"target, arch\" tuple:\n"
    for target in cfg.target:
        ret += str(target) + "\n"
    return ret

def add_developer_options(parser):
    parser.add_argument("-T", dest="log_levels",
                        action="append", default=[],
                        help="[Compiler debugging] Adjust logging level per file (see below)")
    parser.add_argument("--top4", dest="passes",
                        action="append", default=[],
                        help="[Compiler debugging] Dump the P4 representation after \
                               passes whose name contains one of `passX' substrings. \
                               When '-v' is used this will include the compiler IR.")
    parser.add_argument("--dump", dest="dump_dir", default=None,
                        help="[Compiler debugging] Folder where P4 programs are dumped.")
    parser.add_argument("--toJson", dest="json", default=None,
                        help="Dump IR to JSON in the specified file.")
    parser.add_argument("--pp", dest="pretty_print", default=None,
                        help="Pretty-print the program in the specified file.")

def main():
    parser = argparse.ArgumentParser(conflict_handler='resolve')
    parser.add_argument("-V", "--version", dest="show_version",
                        help="show version and exit",
                        action="store_true", default=False)
    parser.add_argument("-v", "--debug", dest="debug",
                        help="verbose",
                        action="store_true", default=False)
    parser.add_argument("-###", "--test-only", dest="dry_run",
                        help="print (but do not run) the commands",
                        action="store_true", default=False)
    parser.add_argument("-Xpreprocessor", dest="preprocessor_options",
                        metavar="<arg>",
                        help="Pass <arg> to the preprocessor",
                        action="append", default=[])
    parser.add_argument("-Xp4c", dest="compiler_options",
                        metavar="<arg>",
                        help="Pass <arg> to the compiler",
                        action="append", default=[])
    parser.add_argument("-Xassembler", dest="assembler_options",
                        metavar="<arg>",
                        help="Pass <arg> to the assembler",
                        action="append", default=[])
    parser.add_argument("-Xlinker", dest="linker_options",
                        metavar="<arg>",
                        help="Pass <arg> to the linker",
                        action="append", default=[])
    parser.add_argument("-b", "--target", dest="target",
                        help="specify target device",
                        action="store", default="bmv2")
    parser.add_argument("-a", "--arch", dest="arch",
                        help="specify target architecture",
                        action="store", default="v1model")
    parser.add_argument("-c", dest="run_all",
                        help="Only run preprocess, compile, and assemble steps",
                        action="store_true", default=True)
    parser.add_argument("-D", dest="preprocessor_defines",
                        help="define a macro to be used by the preprocessor",
                        action="append", default=[])
    parser.add_argument("-E", dest="run_preprocessor_only",
                        help="Only run the preprocessor",
                        action="store_true", default=False)
    parser.add_argument("-e", dest="skip_preprocessor",
                        help="Skip the preprocessor",
                        action="store_true", default=False)
    parser.add_argument("-g", dest="debug_info",
                        help="Generate debug information",
                        action="store_true", default=False)
    parser.add_argument("-I", dest="search_path",
                        help="Add directory to include search path",
                        action="append", default=[])
    parser.add_argument("-o", dest="output_directory",
                        help="Write output to the provided path",
                        action="store", metavar="PATH", default=".")
    parser.add_argument("--p4runtime-file",
                        help="Write a P4Runtime control plane API description "
                        "to the specified file.",
                        action="store", default=None)
    parser.add_argument("--p4runtime-format",
                        choices=["binary", "json", "text"],
                        help="Choose output format for the P4Runtime API "
                        "description (default is binary).",
                        action="store", default="binary")
    parser.add_argument("--target-help", dest="show_target_help",
                        help="Display target specific command line options.",
                        action="store_true", default=False)
    parser.add_argument("-S", dest="run_till_assembler",
                        help="Only run the preprocess and compilation steps",
                        action="store_true", default=False)
    parser.add_argument("--std", "-x", dest="language",
                        choices = ["p4-14", "p4-16"],
                        help="Treat subsequent input files as having type language.",
                        action="store", default="p4-16")

    if (os.environ['P4C_BUILD_TYPE'] == "DEVELOPER"):
        add_developer_options(parser)

    parser.add_argument("source_file", nargs='?', help="Files to compile", default=None)

    # load supported configuration.
    # We load these before we parse options, so that backends can register
    # proprietary options
    cfg_files = glob.glob("{}/*.cfg".format(os.environ['P4C_CFG_PATH']))
    cfg = config.Config(config_prefix = "p4c")
    for cf in cfg_files:
        cfg.load_from_config(cf, parser)

    # parse the arguments
    opts = parser.parse_args()

    user_defined_version = os.environ.get('P4C_DEFAULT_VERSION')
    if user_defined_version != None:
        opts.language = user_defined_version

    user_defined_target = os.environ.get('P4C_DEFAULT_TARGET')
    if user_defined_target != None:
        opts.target = user_defined_target

    user_defined_arch = os.environ.get('P4C_DEFAULT_ARCH')
    if user_defined_arch != None:
        opts.arch = user_defined_arch

    # deal with early exits
    if opts.show_version:
        print "p4c", get_version()
        sys.exit(0)

    if opts.show_target_help:
        print display_supported_targets(cfg)
        sys.exit(0)

    if not opts.source_file:
        parser.error('No input specified.')

    if not os.path.isfile(opts.source_file):
        print >> sys.stderr, 'Input file ' + opts.source_file + ' does not exist'
        sys.exit(1)

    # check that the tuple value is correct
    backend = (opts.target, opts.arch)
    if (len(backend) != 2):
        parser.error("Invalid target and arch tuple: {}\n{}".\
                     format(backend, display_supported_targets(cfg)))

    # find the backend
    backend = None
    for target in cfg.target:
        regex = target._backend.replace('*', '[a-zA-Z0-9*]*')
        pattern = re.compile(regex)
        if (pattern.match(opts.target + '-' + opts.arch)):
            backend = target
            break
    if backend == None:
        parser.error("Unknown backend: {}-{}".format(str(opts.target), str(opts.arch)))

    # set all configuration and command line options for backend
    backend.process_command_line_options(opts)
    # run all commands
    rc = backend.run()
    sys.exit(rc)
<|endoftext|>


<commit_before># This script generates polyamri.h at the top level of the build directory.
import sys, os, os.path
source_dir = sys.argv[1]
target_dir = sys.argv[2]

def visit_c_files(headers, dirname, files):
    if 'tests' in files:
        files.remove('tests')
    h_files = [f for f in files if f[-2:] == '.h' and f[0] != '.']
    headers.extend(h_files)

def find_c_headers(dirname):
    headers = []
    os.path.walk(dirname, visit_c_files, headers)
    return headers

# Generate the top-level header.
c_headers = find_c_headers('%s/polyamri'%source_dir)
header_name = '%s/polyamri.h'%target_dir
if not os.path.exists(target_dir):
    os.makedirs(target_dir)
header = open(header_name, 'w')
header.write('// polyamri.h -- automatically generated.\n')
header.write('// This file is part of the polyamri HPC library. See the license\n')
header.write('// in the actual source files for details of distribution.\n\n')
header.write('#ifndef POLYAMRI_LIBRARY_H\n')
header.write('#define POLYAMRI_LIBRARY_H\n\n')
header.write('#ifdef __cplusplus\n')
header.write('extern "C" {\n')
header.write('#endif\n\n')
header.write('#include "polyamri/polyamri.h"\n')
for h in c_headers:
    header.write('#include "polyamri/%s"\n'%h)
header.write('\n')
header.write('#ifdef __cplusplus\n')
header.write('}\n')
header.write('#endif\n\n')
header.write('#endif\n\n')
header.close()

<commit_msg>Slight fix for double include.<commit_after># This script generates polyamri.h at the top level of the build directory.
import sys, os, os.path
source_dir = sys.argv[1]
target_dir = sys.argv[2]

def visit_c_files(headers, dirname, files):
    if 'tests' in files:
        files.remove('tests')
    h_files = [f for f in files if f[-2:] == '.h' and f[0] != '.']
    headers.extend(h_files)

def find_c_headers(dirname):
    headers = []
    os.path.walk(dirname, visit_c_files, headers)
    return headers

# Generate the top-level header.
c_headers = find_c_headers('%s/polyamri'%source_dir)
header_name = '%s/polyamri.h'%target_dir
if not os.path.exists(target_dir):
    os.makedirs(target_dir)
header = open(header_name, 'w')
header.write('// polyamri.h -- automatically generated.\n')
header.write('// This file is part of the polyamri HPC library. See the license\n')
header.write('// in the actual source files for details of distribution.\n\n')
header.write('#ifndef POLYAMRI_LIBRARY_H\n')
header.write('#define POLYAMRI_LIBRARY_H\n\n')
header.write('#ifdef __cplusplus\n')
header.write('extern "C" {\n')
header.write('#endif\n\n')
header.write('#include "polyamri/polyamri.h"\n')
for h in c_headers:
    if h != 'polyamri.h':
        header.write('#include "polyamri/%s"\n'%h)
header.write('\n')
header.write('#ifdef __cplusplus\n')
header.write('}\n')
header.write('#endif\n\n')
header.write('#endif\n\n')
header.close()

<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import logging
import shutil
import time
import os

def create_test_directory():
    test_dir_name = "tests/" + time.strftime("%Y_%m_%d_%H_%M_%S")
    path = os.path.join(os.getcwd(), test_dir_name)
    os.mkdir(path, mode=0o777)
    print("Test directory: ", path)
    return path

def configure_logging(log_filename, clear_log_file=True):
    format_template = ("%(asctime)s %(threadName)s %(name)s %(levelname)s "
                       "%(filename)-25s %(lineno)-5s "
                       "%(funcName)-25s : %(message)s")
    logging.basicConfig(format=format_template,
                        filename=log_filename,
                        filemode='a',
                        level=logging.DEBUG)
    if clear_log_file:
        with open(log_filename, "w") as f:
            f.write("asctime\t\t\t\t\tthreadName name levelname filename\
            \tlineno\tfuncName\t\t\t\tmessage\n")

    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("matplotlib").setLevel(logging.WARNING)

def copy_config_files_to_test_directory(files: list, test_directory: str):
    for file in files:
        shutil.copy(file, test_directory + "/" + file)

def copy_log_files_to_test_directory(dir: str):
    log_files = ["log/log_rx.log", "log/log_tx.log", "log/check_addr.log"]
    for file in log_files:
        shutil.copy(file, dir + "/" + time.strftime("%Y_%m_%d_%H_%M_%S_") +
                                file.replace("log/", ""))

# Running tests as sudo implies root permissions on created directories/files.
# This function sets the default permission mode to dirs/files in given path
# recursively.
def set_default_chmod_recurs(path):
    for root, dirs, files in os.walk(path):
        for d in dirs:
            os.chmod(os.path.join(root, d), 0o0777)
        for f in files:
            os.chmod(os.path.join(root, f), 0o0777)
<commit_msg>tools/hci_throughput: util path fix<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import logging
import shutil
import time
import os
from pathlib import Path

def create_test_directory():
    test_dir_name = "tests/" + time.strftime("%Y_%m_%d_%H_%M_%S")
    path = os.path.join(os.getcwd(), test_dir_name)
    os.mkdir(path, mode=0o777)
    print("Test directory: ", path)
    return path

def configure_logging(log_filename, clear_log_file=True):
    format_template = ("%(asctime)s %(threadName)s %(name)s %(levelname)s "
                       "%(filename)-25s %(lineno)-5s "
                       "%(funcName)-25s : %(message)s")
    logging.basicConfig(format=format_template,
                        filename=log_filename,
                        filemode='a',
                        level=logging.DEBUG)
    if clear_log_file:
        with open(log_filename, "w") as f:
            f.write("asctime\t\t\t\t\tthreadName name levelname filename\
            \tlineno\tfuncName\t\t\t\tmessage\n")

    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("matplotlib").setLevel(logging.WARNING)

def copy_config_files_to_test_directory(files: list, test_directory: str):
    for file in files:
        shutil.copy(file, test_directory + "/" + Path(file).name)

def copy_log_files_to_test_directory(dir: str):
    log_files = ["log/log_rx.log", "log/log_tx.log", "log/check_addr.log"]
    for file in log_files:
        shutil.copy(file, dir + "/" + time.strftime("%Y_%m_%d_%H_%M_%S_") +
                                file.replace("log/", ""))

# Running tests as sudo implies root permissions on created directories/files.
# This function sets the default permission mode to dirs/files in given path
# recursively.
def set_default_chmod_recurs(path):
    for root, dirs, files in os.walk(path):
        for d in dirs:
            os.chmod(os.path.join(root, d), 0o0777)
        for f in files:
            os.chmod(os.path.join(root, f), 0o0777)
<|endoftext|>


<commit_before>#!/usr/bin/env python3.6
"""Visualise the output of Heinz.

This script is used to visualize the output of Heinz, which is in the form of
DOT language:
    1. Clear the output of Heinz, extract the DOT source code.
    2. Visualize the DOT source code and save it into file.

The function of this script is rather simple, for more advanced visualization,
please adopt other solutions mentioned in the paper
doi: 10.1093/bioinformatics/btv526

This tool is only designed for visualizing the output of Heinz tool.
"""

# Author: Cico Zhang
# Date: 2 Aug 2017
# Version: 0.2

import argparse
import sys

from graphviz import Source

def get_args():
    """Collect the inputs."""
    parser = argparse.ArgumentParser(
        description='Visualise the output of Heinz')
    parser.add_argument('-i', '--input', required=True, dest='heinz',
                        metavar='Heinz_output.txt', type=str,
                        help='Output file of Heinz as the input')
    parser.add_argument('-o', '--output', required=True, dest='output',
                        metavar='graph.pdf', type=str,
                        help='The output file that saves the visualisation')
    args = parser.parse_args()

    if args.heinz is None:
        sys.exit('Input file must be designated.')

    return args

def main():
    """Main function."""
    args = get_args()
    # Read the whole output file
    with open(args.heinz) as r:
        graph_dot = r.readlines()

    # Remove the redundant lines
    while not graph_dot[0].startswith('graph G {'):
        graph_dot.pop(0)

    src = Source(''.join(graph_dot))
    data_pdf = src.pipe('pdf')
    # Redirect the output (very important)
    with open(args.output, 'wb') as w:
        w.write(data_pdf)
    print('The visualization is saved as PDF!')
    sys.exit(0)

if __name__ == "__main__":
    main()
<commit_msg>fix shebang<commit_after>#!/usr/bin/env python
"""Visualise the output of Heinz.

This script is used to visualize the output of Heinz, which is in the form of
DOT language:
    1. Clear the output of Heinz, extract the DOT source code.
    2. Visualize the DOT source code and save it into file.

The function of this script is rather simple, for more advanced visualization,
please adopt other solutions mentioned in the paper
doi: 10.1093/bioinformatics/btv526

This tool is only designed for visualizing the output of Heinz tool.
"""

# Author: Cico Zhang
# Date: 2 Aug 2017
# Version: 0.2

import argparse
import sys

from graphviz import Source

def get_args():
    """Collect the inputs."""
    parser = argparse.ArgumentParser(
        description='Visualise the output of Heinz')
    parser.add_argument('-i', '--input', required=True, dest='heinz',
                        metavar='Heinz_output.txt', type=str,
                        help='Output file of Heinz as the input')
    parser.add_argument('-o', '--output', required=True, dest='output',
                        metavar='graph.pdf', type=str,
                        help='The output file that saves the visualisation')
    args = parser.parse_args()

    if args.heinz is None:
        sys.exit('Input file must be designated.')

    return args

def main():
    """Main function."""
    args = get_args()
    # Read the whole output file
    with open(args.heinz) as r:
        graph_dot = r.readlines()

    # Remove the redundant lines
    while not graph_dot[0].startswith('graph G {'):
        graph_dot.pop(0)

    src = Source(''.join(graph_dot))
    data_pdf = src.pipe('pdf')
    # Redirect the output (very important)
    with open(args.output, 'wb') as w:
        w.write(data_pdf)
    print('The visualization is saved as PDF!')
    sys.exit(0)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from PyQt4 import QtCore, QtGui, uic
from PyQt4.QtCore import *
from PyQt4.QtGui import *

class Mod:
    def __init__(self, path):
        import os
        self.path = path
        self.size = None
        self.name = None

        if os.path.exists("%s/%s.txt" % (path, os.path.basename(path))):
            self.name = os.path.basename(path)

def isZipFile(path):
    import zipfile
    return zipfile.is_zipfile(path)

def showError(error):
    print error

class MyTable(QtGui.QTableWidget):
    def __init__(self, parent):
        QtGui.QTableWidget.__init__(self, parent)
        self.setAcceptDrops(True)

    def initialize(self):
        self.addMod('/home/jon/.paintown/wolfburg')

    def addMod(self, path):
        column_name = 0
        column_enabled = 1
        column_size = 2

        mod = Mod(path)
        row = self.rowCount()
        self.setRowCount(self.rowCount() + 1)
        self.setColumnCount(3)

        self.setItem(row, column_name, QtGui.QTableWidgetItem(mod.name))
        # self.setItem(1, 1, QtGui.QTableWidgetItem('foobar'))
        #for c in range(0, self.columnCount()):
        #    for r in range(0, self.rowCount()):
        #        self.setItem(r, c, QtGui.QTableWidgetItem('foobar'))

    def dragEnterEvent(self, event):
        # print "Has urls? %s" % event.mimeData().hasUrls()
        if event.mimeData().hasUrls():
            # print "Accept event"
            event.accept()
        else:
            event.ignore()
        # print "Drag event: mime %s" % event.mimeData().urls()

    def dragMoveEvent(self, event):
        event.accept()
        # print "Drag move event"

    def doFile(self, path):
        if isZipFile(path):
            print "Installing mod %s" % path

    def dropEvent(self, event):
        for file in event.mimeData().urls():
            path = file.path()
            if isZipFile(path):
                self.doFile(path)
            else:
                showError("'%s' is not a zip file. Please drag and drop .zip files" % path)
<commit_msg>search for mods<commit_after>from PyQt4 import QtCore, QtGui, uic
from PyQt4.QtCore import *
from PyQt4.QtGui import *

class Mod:
    def __init__(self, path):
        import os
        self.path = path
        self.size = None
        self.name = os.path.basename(path)

def isZipFile(path):
    import zipfile
    return zipfile.is_zipfile(path)

def showError(error):
    print error

modPath = '/home/jon/.paintown'

class MyTable(QtGui.QTableWidget):
    def __init__(self, parent):
        QtGui.QTableWidget.__init__(self, parent)
        self.setAcceptDrops(True)

    def initialize(self):
        def maybeAdd(path):
            import os
            #print "Check path %s" % path
            if os.path.exists("%s/%s.txt" % (path, os.path.basename(path))):
                self.addMod(path)

        import os
        for path in os.listdir(modPath):
            maybeAdd('%s/%s' % (modPath, path))

    def addMod(self, path):
        column_name = 0
        column_enabled = 1
        column_size = 2

        mod = Mod(path)
        row = self.rowCount()
        self.setRowCount(self.rowCount() + 1)
        self.setColumnCount(3)

        self.setItem(row, column_name, QtGui.QTableWidgetItem(mod.name))
        # self.setItem(1, 1, QtGui.QTableWidgetItem('foobar'))
        #for c in range(0, self.columnCount()):
        #    for r in range(0, self.rowCount()):
        #        self.setItem(r, c, QtGui.QTableWidgetItem('foobar'))

    def dragEnterEvent(self, event):
        # print "Has urls? %s" % event.mimeData().hasUrls()
        if event.mimeData().hasUrls():
            # print "Accept event"
            event.accept()
        else:
            event.ignore()
        # print "Drag event: mime %s" % event.mimeData().urls()

    def dragMoveEvent(self, event):
        event.accept()
        # print "Drag move event"

    def doFile(self, path):
        if isZipFile(path):
            print "Installing mod %s" % path

    def dropEvent(self, event):
        for file in event.mimeData().urls():
            path = file.path()
            if isZipFile(path):
                self.doFile(path)
            else:
                showError("'%s' is not a zip file. Please drag and drop .zip files" % path)
<|endoftext|>